From a794490b643af03d01be581734b685a78b07f759 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 16 Jan 2025 16:42:06 +0100 Subject: [PATCH 01/29] feat: Read variables from file --- bundle/config/root.go | 16 ++++++++++ bundle/config/root_test.go | 50 ++++++++++++++++++++++++++++++ bundle/config/variable/variable.go | 8 +++-- cmd/bundle/utils/utils.go | 42 +++++++++++++++++++++++-- cmd/bundle/variables.go | 1 + 5 files changed, 112 insertions(+), 5 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 21804110a6..89c9b22333 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -260,6 +260,22 @@ func (r *Root) InitializeVariables(vars []string) error { return nil } +// Initializes variables parsed from variable file +// Variables can have any type of value, including complex types +func (r *Root) InitializeAnyTypeVariables(vars map[string]any) error { + for name, val := range vars { + if _, ok := r.Variables[name]; !ok { + return fmt.Errorf("variable %s has not been defined", name) + } + + err := r.Variables[name].Set(val) + if err != nil { + return fmt.Errorf("failed to assign %s to %s: %s", val, name, err) + } + } + return nil +} + func (r *Root) Merge(other *Root) error { // Merge dynamic configuration values. return r.Mutate(func(root dyn.Value) (dyn.Value, error) { diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index 42fae49d98..27339f997b 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -51,6 +51,56 @@ func TestInitializeVariables(t *testing.T) { assert.Equal(t, "456", (root.Variables["bar"].Value)) } +func TestInitializeAnyTypeVariables(t *testing.T) { + root := &Root{ + Variables: map[string]*variable.Variable{ + "string": { + Default: "default", + Description: "string variable", + }, + "int": { + Default: 0, + Description: "int variable", + }, + "complex": { + Default: []map[string]int{{"a": 1}, {"b": 2}}, + Description: "complex variable", + Type: variable.VariableTypeComplex, + }, + "unused": { + Default: "should remain default", + Description: "unused variable", + }, + }, + } + + err := root.InitializeAnyTypeVariables(map[string]any{ + "string": "value", + "int": 1, + "complex": []map[string]int{{"c": 3}}, + }) + assert.NoError(t, err) + assert.Equal(t, "value", (root.Variables["string"].Value)) + assert.Equal(t, 1, (root.Variables["int"].Value)) + assert.Equal(t, []map[string]int{{"c": 3}}, (root.Variables["complex"].Value)) +} + +func TestInitializeAnyTypeVariablesUndeclared(t *testing.T) { + root := &Root{ + Variables: map[string]*variable.Variable{ + "string": { + Default: "default", + Description: "string variable", + }, + }, + } + + err := root.InitializeAnyTypeVariables(map[string]any{ + "not_declared": "value", + }) + assert.ErrorContains(t, err, "variable not_declared has not been defined") +} + func TestInitializeVariablesWithAnEqualSignInValue(t *testing.T) { root := &Root{ Variables: map[string]*variable.Variable{ diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index 95a68cfeb6..34f58f19fe 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -36,9 +36,11 @@ type Variable struct { // This field stores the resolved value for the variable. The variable are // resolved in the following priority order (from highest to lowest) // - // 1. Command line flag. For example: `--var="foo=bar"` - // 2. Target variable. eg: BUNDLE_VAR_foo=bar - // 3. Default value as defined in the applicable environments block + // 1. Command line flag, one of these is used + // a. Variable value obtained from arguments, example: `--var="foo=bar"` + // b. Variable value obtained from the file, example: `--vars-file-path="/path/to/file"` + // 2. Environment variable. eg: BUNDLE_VAR_foo=bar + // 3. Default value as defined in the applicable targets block // 4. Default value defined in variable definition // 5. Throw error, since if no default value is defined, then the variable // is required diff --git a/cmd/bundle/utils/utils.go b/cmd/bundle/utils/utils.go index ce3774cf55..fca19e03e1 100644 --- a/cmd/bundle/utils/utils.go +++ b/cmd/bundle/utils/utils.go @@ -2,6 +2,9 @@ package utils import ( "context" + "encoding/json" + "fmt" + "os" "github.com/databricks/cli/bundle" "github.com/databricks/cli/cmd/root" @@ -16,6 +19,30 @@ func configureVariables(cmd *cobra.Command, b *bundle.Bundle, variables []string }) } +func configureVariablesFromFile(cmd *cobra.Command, b *bundle.Bundle, filePath string) diag.Diagnostics { + return bundle.ApplyFunc(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + f, err := os.ReadFile(filePath) + if err != nil { + return diag.FromErr(fmt.Errorf("failed to read variables file: %w", err)) + } + + vars := map[string]any{} + err = json.Unmarshal(f, &vars) + if err != nil { + return diag.FromErr(fmt.Errorf("failed to parse variables file: %w", err)) + } + + if len(vars) > 0 { + err = b.Config.InitializeAnyTypeVariables(vars) + if err != nil { + return diag.FromErr(err) + } + } + + return nil + }) +} + func ConfigureBundleWithVariables(cmd *cobra.Command) (*bundle.Bundle, diag.Diagnostics) { // Load bundle config and apply target b, diags := root.MustConfigureBundle(cmd) @@ -27,9 +54,20 @@ func ConfigureBundleWithVariables(cmd *cobra.Command) (*bundle.Bundle, diag.Diag if err != nil { return b, diag.FromErr(err) } + variableFilePath, err := cmd.Flags().GetString("vars-file-path") + if err != nil { + return b, diag.FromErr(err) + } - // Initialize variables by assigning them values passed as command line flags - diags = diags.Extend(configureVariables(cmd, b, variables)) + if len(variables) > 0 && variableFilePath != "" { + return b, diag.Errorf("cannot specify both --var and --vars-file-path flags") + } else if len(variables) > 0 { + // Initialize variables by assigning them values passed as command line flags + diags = diags.Extend(configureVariables(cmd, b, variables)) + } else if variableFilePath != "" { + // Initialize variables by loading them from a file + diags = diags.Extend(configureVariablesFromFile(cmd, b, variableFilePath)) + } return b, diags } diff --git a/cmd/bundle/variables.go b/cmd/bundle/variables.go index f8f5167ead..8251e28fa0 100644 --- a/cmd/bundle/variables.go +++ b/cmd/bundle/variables.go @@ -6,4 +6,5 @@ import ( func initVariableFlag(cmd *cobra.Command) { cmd.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in bundle config. Example: --var="foo=bar"`) + cmd.PersistentFlags().String("vars-file-path", "", `file path to a JSON file containing variables. Example: --vars-file-path="/path/to/vars.json"`) } From 3f0b5f848ec65075e18d99d46212ffbff42c97d8 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 16 Jan 2025 18:07:51 +0100 Subject: [PATCH 02/29] test: Acceptance test for flag overrides --- .../variables/flag_overrides/databricks.yml | 21 +++++++++++++++++++ .../variables/flag_overrides/output.txt | 8 +++++++ .../bundle/variables/flag_overrides/script | 5 +++++ .../bundle/variables/flag_overrides/vars.json | 7 +++++++ 4 files changed, 41 insertions(+) create mode 100644 acceptance/bundle/variables/flag_overrides/databricks.yml create mode 100644 acceptance/bundle/variables/flag_overrides/output.txt create mode 100644 acceptance/bundle/variables/flag_overrides/script create mode 100644 acceptance/bundle/variables/flag_overrides/vars.json diff --git a/acceptance/bundle/variables/flag_overrides/databricks.yml b/acceptance/bundle/variables/flag_overrides/databricks.yml new file mode 100644 index 0000000000..41c106ac6d --- /dev/null +++ b/acceptance/bundle/variables/flag_overrides/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: TestResolveVariablesFromFile + +variables: + cluster: + type: "complex" + default: + node_type_id: "unused" + cluster_key: + default: "unused" + cluster_workers: + default: 1 + +resources: + jobs: + job1: + job_clusters: + - job_cluster_key: ${var.cluster_key} + new_cluster: + node_type_id: "${var.cluster.node_type_id}" + num_workers: ${var.cluster_workers} diff --git a/acceptance/bundle/variables/flag_overrides/output.txt b/acceptance/bundle/variables/flag_overrides/output.txt new file mode 100644 index 0000000000..549cf9e368 --- /dev/null +++ b/acceptance/bundle/variables/flag_overrides/output.txt @@ -0,0 +1,8 @@ +{ + "job_cluster_key": "mlops_stacks-cluster", + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 + } +} +"mlops_stacks-cluster" diff --git a/acceptance/bundle/variables/flag_overrides/script b/acceptance/bundle/variables/flag_overrides/script new file mode 100644 index 0000000000..afd3418d84 --- /dev/null +++ b/acceptance/bundle/variables/flag_overrides/script @@ -0,0 +1,5 @@ +# variable file +$CLI bundle validate -o json --vars-file-path=vars.json | jq .resources.jobs.job1.job_clusters[0] + +# variable flag +$CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq .resources.jobs.job1.job_clusters[0].job_cluster_key diff --git a/acceptance/bundle/variables/flag_overrides/vars.json b/acceptance/bundle/variables/flag_overrides/vars.json new file mode 100644 index 0000000000..3a865e120e --- /dev/null +++ b/acceptance/bundle/variables/flag_overrides/vars.json @@ -0,0 +1,7 @@ +{ + "cluster": { + "node_type_id": "Standard_DS3_v2" + }, + "cluster_key": "mlops_stacks-cluster", + "cluster_workers": 2 +} From a75c6afd6bdb28d807d37465b5a11866fac1d448 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 17 Jan 2025 11:40:35 +0100 Subject: [PATCH 03/29] fix: Rename to --var-file --- acceptance/bundle/variables/flag_overrides/script | 2 +- bundle/config/variable/variable.go | 2 +- cmd/bundle/utils/utils.go | 4 ++-- cmd/bundle/variables.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/variables/flag_overrides/script b/acceptance/bundle/variables/flag_overrides/script index afd3418d84..bfdbdf427b 100644 --- a/acceptance/bundle/variables/flag_overrides/script +++ b/acceptance/bundle/variables/flag_overrides/script @@ -1,5 +1,5 @@ # variable file -$CLI bundle validate -o json --vars-file-path=vars.json | jq .resources.jobs.job1.job_clusters[0] +$CLI bundle validate -o json --var-file=vars.json | jq .resources.jobs.job1.job_clusters[0] # variable flag $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq .resources.jobs.job1.job_clusters[0].job_cluster_key diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index 34f58f19fe..76feebaad7 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -38,7 +38,7 @@ type Variable struct { // // 1. Command line flag, one of these is used // a. Variable value obtained from arguments, example: `--var="foo=bar"` - // b. Variable value obtained from the file, example: `--vars-file-path="/path/to/file"` + // b. Variable value obtained from the file, example: `--var-file="/path/to/file"` // 2. Environment variable. eg: BUNDLE_VAR_foo=bar // 3. Default value as defined in the applicable targets block // 4. Default value defined in variable definition diff --git a/cmd/bundle/utils/utils.go b/cmd/bundle/utils/utils.go index fca19e03e1..f46d467df6 100644 --- a/cmd/bundle/utils/utils.go +++ b/cmd/bundle/utils/utils.go @@ -54,13 +54,13 @@ func ConfigureBundleWithVariables(cmd *cobra.Command) (*bundle.Bundle, diag.Diag if err != nil { return b, diag.FromErr(err) } - variableFilePath, err := cmd.Flags().GetString("vars-file-path") + variableFilePath, err := cmd.Flags().GetString("var-file") if err != nil { return b, diag.FromErr(err) } if len(variables) > 0 && variableFilePath != "" { - return b, diag.Errorf("cannot specify both --var and --vars-file-path flags") + return b, diag.Errorf("cannot specify both --var and --var-file flags") } else if len(variables) > 0 { // Initialize variables by assigning them values passed as command line flags diags = diags.Extend(configureVariables(cmd, b, variables)) diff --git a/cmd/bundle/variables.go b/cmd/bundle/variables.go index 8251e28fa0..e409e8946c 100644 --- a/cmd/bundle/variables.go +++ b/cmd/bundle/variables.go @@ -6,5 +6,5 @@ import ( func initVariableFlag(cmd *cobra.Command) { cmd.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in bundle config. Example: --var="foo=bar"`) - cmd.PersistentFlags().String("vars-file-path", "", `file path to a JSON file containing variables. Example: --vars-file-path="/path/to/vars.json"`) + cmd.PersistentFlags().String("var-file", "", `file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json"`) } From 66dae816ab378cda5eb5cafa2c6d55d9f1226e2b Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 17 Jan 2025 12:04:50 +0100 Subject: [PATCH 04/29] fix: More detailed error for json parsing --- cmd/bundle/utils/utils.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/cmd/bundle/utils/utils.go b/cmd/bundle/utils/utils.go index f46d467df6..f58ef274ce 100644 --- a/cmd/bundle/utils/utils.go +++ b/cmd/bundle/utils/utils.go @@ -2,13 +2,14 @@ package utils import ( "context" - "encoding/json" "fmt" "os" "github.com/databricks/cli/bundle" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/jsonloader" "github.com/spf13/cobra" ) @@ -20,18 +21,29 @@ func configureVariables(cmd *cobra.Command, b *bundle.Bundle, variables []string } func configureVariablesFromFile(cmd *cobra.Command, b *bundle.Bundle, filePath string) diag.Diagnostics { + var diags diag.Diagnostics return bundle.ApplyFunc(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { f, err := os.ReadFile(filePath) if err != nil { return diag.FromErr(fmt.Errorf("failed to read variables file: %w", err)) } - vars := map[string]any{} - err = json.Unmarshal(f, &vars) + val, err := jsonloader.LoadJSON(f, filePath) if err != nil { return diag.FromErr(fmt.Errorf("failed to parse variables file: %w", err)) } + vars := map[string]any{} + err = convert.ToTyped(&vars, val) + if err != nil { + return diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "failed to parse variables file: " + err.Error(), + Detail: "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}", + Locations: val.Locations(), + }) + } + if len(vars) > 0 { err = b.Config.InitializeAnyTypeVariables(vars) if err != nil { From d0db1d73f95022b20c3d49dd015ae5fd3ff46592 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 17 Jan 2025 12:13:09 +0100 Subject: [PATCH 05/29] fix: Cleanup --- bundle/config/root.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 89c9b22333..c5e52ec2e4 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -264,11 +264,12 @@ func (r *Root) InitializeVariables(vars []string) error { // Variables can have any type of value, including complex types func (r *Root) InitializeAnyTypeVariables(vars map[string]any) error { for name, val := range vars { - if _, ok := r.Variables[name]; !ok { + variable, ok := r.Variables[name] + if !ok { return fmt.Errorf("variable %s has not been defined", name) } - err := r.Variables[name].Set(val) + err := variable.Set(val) if err != nil { return fmt.Errorf("failed to assign %s to %s: %s", val, name, err) } From 2c9adcb1edd6ac9e7e67572c4cd355ec478e679f Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 17 Jan 2025 18:32:38 +0100 Subject: [PATCH 06/29] fix: More acceptance tests --- acceptance/bundle/variables/empty/output.txt | 2 +- .../bundle/variables/env_overrides/output.txt | 2 +- .../variables/flag_overrides/output.txt | 8 -- .../bundle/variables/flag_overrides/script | 5 - .../bundle/variables/vanilla/output.txt | 2 +- .../var_file_overrides/databricks.yml | 32 +++++ .../variables/var_file_overrides/output.txt | 126 ++++++++++++++++++ .../variables/var_file_overrides/script | 31 +++++ .../var_files/complex_to_string.json | 3 + .../var_file_overrides/var_files/empty.json | 1 + .../var_files/invalid_json.json | 1 + .../var_files/normal.json} | 0 .../var_files/string_to_complex.json | 5 + .../var_files/undeclared.json | 3 + .../var_files/wrong_file_structure.json | 3 + .../databricks.yml | 6 +- .../bundle/variables/var_flag/output.txt | 3 + acceptance/bundle/variables/var_flag/script | 2 + bundle/config/mutator/set_variables.go | 2 +- 19 files changed, 217 insertions(+), 20 deletions(-) delete mode 100644 acceptance/bundle/variables/flag_overrides/output.txt delete mode 100644 acceptance/bundle/variables/flag_overrides/script create mode 100644 acceptance/bundle/variables/var_file_overrides/databricks.yml create mode 100644 acceptance/bundle/variables/var_file_overrides/output.txt create mode 100644 acceptance/bundle/variables/var_file_overrides/script create mode 100644 acceptance/bundle/variables/var_file_overrides/var_files/complex_to_string.json create mode 100644 acceptance/bundle/variables/var_file_overrides/var_files/empty.json create mode 100644 acceptance/bundle/variables/var_file_overrides/var_files/invalid_json.json rename acceptance/bundle/variables/{flag_overrides/vars.json => var_file_overrides/var_files/normal.json} (100%) create mode 100644 acceptance/bundle/variables/var_file_overrides/var_files/string_to_complex.json create mode 100644 acceptance/bundle/variables/var_file_overrides/var_files/undeclared.json create mode 100644 acceptance/bundle/variables/var_file_overrides/var_files/wrong_file_structure.json rename acceptance/bundle/variables/{flag_overrides => var_flag}/databricks.yml (78%) create mode 100644 acceptance/bundle/variables/var_flag/output.txt create mode 100644 acceptance/bundle/variables/var_flag/script diff --git a/acceptance/bundle/variables/empty/output.txt b/acceptance/bundle/variables/empty/output.txt index 2616359206..589a0a5028 100644 --- a/acceptance/bundle/variables/empty/output.txt +++ b/acceptance/bundle/variables/empty/output.txt @@ -1,4 +1,4 @@ -Error: no value assigned to required variable a. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_a environment variable +Error: no value assigned to required variable a. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_a environment variable Name: empty${var.a} Target: default diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt index f42f82211a..faa3505f36 100644 --- a/acceptance/bundle/variables/env_overrides/output.txt +++ b/acceptance/bundle/variables/env_overrides/output.txt @@ -9,7 +9,7 @@ "prod-a env-var-b" >>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment -Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable +Error: no value assigned to required variable b. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_b environment variable Name: test bundle Target: env-missing-a-required-variable-assignment diff --git a/acceptance/bundle/variables/flag_overrides/output.txt b/acceptance/bundle/variables/flag_overrides/output.txt deleted file mode 100644 index 549cf9e368..0000000000 --- a/acceptance/bundle/variables/flag_overrides/output.txt +++ /dev/null @@ -1,8 +0,0 @@ -{ - "job_cluster_key": "mlops_stacks-cluster", - "new_cluster": { - "node_type_id": "Standard_DS3_v2", - "num_workers": 2 - } -} -"mlops_stacks-cluster" diff --git a/acceptance/bundle/variables/flag_overrides/script b/acceptance/bundle/variables/flag_overrides/script deleted file mode 100644 index bfdbdf427b..0000000000 --- a/acceptance/bundle/variables/flag_overrides/script +++ /dev/null @@ -1,5 +0,0 @@ -# variable file -$CLI bundle validate -o json --var-file=vars.json | jq .resources.jobs.job1.job_clusters[0] - -# variable flag -$CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq .resources.jobs.job1.job_clusters[0].job_cluster_key diff --git a/acceptance/bundle/variables/vanilla/output.txt b/acceptance/bundle/variables/vanilla/output.txt index 1d88bd0601..36cf44b5aa 100644 --- a/acceptance/bundle/variables/vanilla/output.txt +++ b/acceptance/bundle/variables/vanilla/output.txt @@ -3,7 +3,7 @@ "abc def" >>> errcode $CLI bundle validate -Error: no value assigned to required variable b. Assignment can be done through the "--var" flag or by setting the BUNDLE_VAR_b environment variable +Error: no value assigned to required variable b. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_b environment variable Name: ${var.a} ${var.b} Target: default diff --git a/acceptance/bundle/variables/var_file_overrides/databricks.yml b/acceptance/bundle/variables/var_file_overrides/databricks.yml new file mode 100644 index 0000000000..cf743a5566 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/databricks.yml @@ -0,0 +1,32 @@ +bundle: + name: TestResolveVariablesFromFile + +variables: + cluster: + type: "complex" + cluster_key: + cluster_workers: + +resources: + jobs: + job1: + job_clusters: + - job_cluster_key: ${var.cluster_key} + new_cluster: + node_type_id: "${var.cluster.node_type_id}" + num_workers: ${var.cluster_workers} + +targets: + with-defaults: + default: true + variables: + cluster_workers: + default: 1 + cluster: + type: "complex" + default: + node_type_id: "default" + cluster_key: + default: "default" + + without-defaults: diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt new file mode 100644 index 0000000000..1f6184397a --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -0,0 +1,126 @@ + +>>> $CLI bundle validate -o json --var-file=var_files/normal.json +{ + "job_cluster_key": "mlops_stacks-cluster", + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 + } +} + +>>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster +{ + "job_cluster_key": "mlops_stacks-cluster", + "new_cluster": { + "node_type_id": "default", + "num_workers": 1 + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var=cluster_key=mlops_stacks-cluster +Error: cannot specify both --var and --var-file flags + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/not_found.json +Error: failed to read variables file: open var_files/not_found.json: no such file or directory + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json +Error: failed to parse variables file: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a') + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/wrong_file_structure.json +Error: failed to parse variables file: var_files/wrong_file_structure.json:1:1: expected a map, found a sequence + in var_files/wrong_file_structure.json:1:1 + +Variables file must be a JSON object with the following format: +{"var1": "value1", "var2": "value2"} + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/undeclared.json +Error: variable undeclared_var has not been defined + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/complex_to_string.json +Error: expected a map to index "variables.cluster.value.node_type_id", found string + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --var-file=var_files/string_to_complex.json +Error: failed to assign map[node_type_id:Standard_DS3_v2] to cluster_key: variable type is not complex + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +>>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/empty.json +Error: no value assigned to required variable cluster_workers. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_cluster_workers environment variable + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} diff --git a/acceptance/bundle/variables/var_file_overrides/script b/acceptance/bundle/variables/var_file_overrides/script new file mode 100644 index 0000000000..147ce5acba --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/script @@ -0,0 +1,31 @@ +cluster_expr=".resources.jobs.job1.job_clusters[0]" + +# variable file +trace $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr + +# variable flag +trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr + +# both variable file and flag +trace errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr + +# file not found +trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json | jq $cluster_expr + +# file cannot be parsed +trace errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json | jq $cluster_expr + +# file has wrong structure +trace errcode $CLI bundle validate -o json --var-file=var_files/wrong_file_structure.json | jq $cluster_expr + +# file has variable name that is not defined +trace errcode $CLI bundle validate -o json --var-file=var_files/undeclared.json | jq $cluster_expr + +# file has variable name that is complex but default is string +trace errcode $CLI bundle validate -o json --var-file=var_files/complex_to_string.json | jq $cluster_expr + +# file has variable name that is string but default is complex +trace errcode $CLI bundle validate -o json --var-file=var_files/string_to_complex.json | jq $cluster_expr + +# variable is required but it's not provided in the file +trace errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/empty.json | jq $cluster_expr diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/complex_to_string.json b/acceptance/bundle/variables/var_file_overrides/var_files/complex_to_string.json new file mode 100644 index 0000000000..1ea719446a --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/complex_to_string.json @@ -0,0 +1,3 @@ +{ + "cluster": "mlops_stacks-cluster" +} diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/empty.json b/acceptance/bundle/variables/var_file_overrides/var_files/empty.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/empty.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/invalid_json.json b/acceptance/bundle/variables/var_file_overrides/var_files/invalid_json.json new file mode 100644 index 0000000000..257cc5642c --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/invalid_json.json @@ -0,0 +1 @@ +foo diff --git a/acceptance/bundle/variables/flag_overrides/vars.json b/acceptance/bundle/variables/var_file_overrides/var_files/normal.json similarity index 100% rename from acceptance/bundle/variables/flag_overrides/vars.json rename to acceptance/bundle/variables/var_file_overrides/var_files/normal.json diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/string_to_complex.json b/acceptance/bundle/variables/var_file_overrides/var_files/string_to_complex.json new file mode 100644 index 0000000000..602567a68d --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/string_to_complex.json @@ -0,0 +1,5 @@ +{ + "cluster_key": { + "node_type_id": "Standard_DS3_v2" + } +} diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/undeclared.json b/acceptance/bundle/variables/var_file_overrides/var_files/undeclared.json new file mode 100644 index 0000000000..2a137d23d4 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/undeclared.json @@ -0,0 +1,3 @@ +{ + "undeclared_var": 1 +} diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/wrong_file_structure.json b/acceptance/bundle/variables/var_file_overrides/var_files/wrong_file_structure.json new file mode 100644 index 0000000000..de140ba36d --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/wrong_file_structure.json @@ -0,0 +1,3 @@ +[ + "foo" +] diff --git a/acceptance/bundle/variables/flag_overrides/databricks.yml b/acceptance/bundle/variables/var_flag/databricks.yml similarity index 78% rename from acceptance/bundle/variables/flag_overrides/databricks.yml rename to acceptance/bundle/variables/var_flag/databricks.yml index 41c106ac6d..f9de114e53 100644 --- a/acceptance/bundle/variables/flag_overrides/databricks.yml +++ b/acceptance/bundle/variables/var_flag/databricks.yml @@ -1,13 +1,13 @@ bundle: - name: TestResolveVariablesFromFile + name: TestResolveVariablesFromFlag variables: cluster: type: "complex" default: - node_type_id: "unused" + node_type_id: "undefined" cluster_key: - default: "unused" + default: "undefined" cluster_workers: default: 1 diff --git a/acceptance/bundle/variables/var_flag/output.txt b/acceptance/bundle/variables/var_flag/output.txt new file mode 100644 index 0000000000..b5390d371c --- /dev/null +++ b/acceptance/bundle/variables/var_flag/output.txt @@ -0,0 +1,3 @@ + +>>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster +"mlops_stacks-cluster" diff --git a/acceptance/bundle/variables/var_flag/script b/acceptance/bundle/variables/var_flag/script new file mode 100644 index 0000000000..66a1217d8d --- /dev/null +++ b/acceptance/bundle/variables/var_flag/script @@ -0,0 +1,2 @@ +# single flag +trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq .resources.jobs.job1.job_clusters[0].job_cluster_key diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index 9e9f2dcfef..7b16571e21 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -64,7 +64,7 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, } // We should have had a value to set for the variable at this point. - return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) + return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" or "--var-file" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) } func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { From 15c9c5c29b62d5707c33b33609f8341612c41fa2 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Sat, 18 Jan 2025 12:58:13 +0100 Subject: [PATCH 07/29] fix: Flaky test --- acceptance/bundle/variables/var_file_overrides/output.txt | 2 +- .../bundle/variables/var_file_overrides/var_files/empty.json | 1 - .../var_file_overrides/var_files/without_required.json | 3 +++ 3 files changed, 4 insertions(+), 2 deletions(-) delete mode 100644 acceptance/bundle/variables/var_file_overrides/var_files/empty.json create mode 100644 acceptance/bundle/variables/var_file_overrides/var_files/without_required.json diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt index 1f6184397a..0cb274fc65 100644 --- a/acceptance/bundle/variables/var_file_overrides/output.txt +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -113,7 +113,7 @@ Exit code: 1 } >>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/empty.json -Error: no value assigned to required variable cluster_workers. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_cluster_workers environment variable +Error: failed to read variables file: open var_files/empty.json: no such file or directory Exit code: 1 diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/empty.json b/acceptance/bundle/variables/var_file_overrides/var_files/empty.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/acceptance/bundle/variables/var_file_overrides/var_files/empty.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/without_required.json b/acceptance/bundle/variables/var_file_overrides/var_files/without_required.json new file mode 100644 index 0000000000..37e52b0fe0 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/var_files/without_required.json @@ -0,0 +1,3 @@ +{ + "cluster_key": "mlops_stacks-cluster" +} From 01a6a6613a678dac863a211b0b3b492c4c40cdb9 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 20 Jan 2025 12:05:08 +0100 Subject: [PATCH 08/29] fix: Remove extra tests --- bundle/config/root_test.go | 50 -------------------------------------- 1 file changed, 50 deletions(-) diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index 27339f997b..42fae49d98 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -51,56 +51,6 @@ func TestInitializeVariables(t *testing.T) { assert.Equal(t, "456", (root.Variables["bar"].Value)) } -func TestInitializeAnyTypeVariables(t *testing.T) { - root := &Root{ - Variables: map[string]*variable.Variable{ - "string": { - Default: "default", - Description: "string variable", - }, - "int": { - Default: 0, - Description: "int variable", - }, - "complex": { - Default: []map[string]int{{"a": 1}, {"b": 2}}, - Description: "complex variable", - Type: variable.VariableTypeComplex, - }, - "unused": { - Default: "should remain default", - Description: "unused variable", - }, - }, - } - - err := root.InitializeAnyTypeVariables(map[string]any{ - "string": "value", - "int": 1, - "complex": []map[string]int{{"c": 3}}, - }) - assert.NoError(t, err) - assert.Equal(t, "value", (root.Variables["string"].Value)) - assert.Equal(t, 1, (root.Variables["int"].Value)) - assert.Equal(t, []map[string]int{{"c": 3}}, (root.Variables["complex"].Value)) -} - -func TestInitializeAnyTypeVariablesUndeclared(t *testing.T) { - root := &Root{ - Variables: map[string]*variable.Variable{ - "string": { - Default: "default", - Description: "string variable", - }, - }, - } - - err := root.InitializeAnyTypeVariables(map[string]any{ - "not_declared": "value", - }) - assert.ErrorContains(t, err, "variable not_declared has not been defined") -} - func TestInitializeVariablesWithAnEqualSignInValue(t *testing.T) { root := &Root{ Variables: map[string]*variable.Variable{ From f52423c135731e829941c1d4416ceacb4e199f51 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 20 Jan 2025 12:10:29 +0100 Subject: [PATCH 09/29] fix: Update `help` snapshots --- acceptance/bundle/help/bundle-deploy/output.txt | 11 ++++++----- acceptance/bundle/help/bundle-deployment/output.txt | 11 ++++++----- acceptance/bundle/help/bundle-destroy/output.txt | 11 ++++++----- .../help/bundle-generate-dashboard/output.txt | 13 +++++++------ .../bundle/help/bundle-generate-job/output.txt | 13 +++++++------ .../bundle/help/bundle-generate-pipeline/output.txt | 13 +++++++------ acceptance/bundle/help/bundle-generate/output.txt | 11 ++++++----- acceptance/bundle/help/bundle-init/output.txt | 11 ++++++----- acceptance/bundle/help/bundle-open/output.txt | 11 ++++++----- acceptance/bundle/help/bundle-run/output.txt | 11 ++++++----- acceptance/bundle/help/bundle-schema/output.txt | 11 ++++++----- acceptance/bundle/help/bundle-summary/output.txt | 11 ++++++----- acceptance/bundle/help/bundle-sync/output.txt | 9 +++++---- acceptance/bundle/help/bundle-validate/output.txt | 11 ++++++----- acceptance/bundle/help/bundle/output.txt | 5 +++-- 15 files changed, 89 insertions(+), 74 deletions(-) diff --git a/acceptance/bundle/help/bundle-deploy/output.txt b/acceptance/bundle/help/bundle-deploy/output.txt index 13c903f3e6..0d06436b9c 100644 --- a/acceptance/bundle/help/bundle-deploy/output.txt +++ b/acceptance/bundle/help/bundle-deploy/output.txt @@ -14,8 +14,9 @@ Flags: -h, --help help for deploy Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle-deployment/output.txt b/acceptance/bundle/help/bundle-deployment/output.txt index ddf5b33054..faae384b07 100644 --- a/acceptance/bundle/help/bundle-deployment/output.txt +++ b/acceptance/bundle/help/bundle-deployment/output.txt @@ -13,10 +13,11 @@ Flags: -h, --help help for deployment Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" Use "databricks bundle deployment [command] --help" for more information about a command. diff --git a/acceptance/bundle/help/bundle-destroy/output.txt b/acceptance/bundle/help/bundle-destroy/output.txt index d701643012..54fac38b49 100644 --- a/acceptance/bundle/help/bundle-destroy/output.txt +++ b/acceptance/bundle/help/bundle-destroy/output.txt @@ -11,8 +11,9 @@ Flags: -h, --help help for destroy Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle-generate-dashboard/output.txt b/acceptance/bundle/help/bundle-generate-dashboard/output.txt index a63ce0ff87..240a5a580e 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/output.txt +++ b/acceptance/bundle/help/bundle-generate-dashboard/output.txt @@ -16,9 +16,10 @@ Flags: --watch watch for changes to the dashboard and update the configuration Global Flags: - --debug enable debug logging - --key string resource key to use for the generated configuration - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + --key string resource key to use for the generated configuration + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle-generate-job/output.txt b/acceptance/bundle/help/bundle-generate-job/output.txt index adc3f45ae5..6bb4fc2c8d 100644 --- a/acceptance/bundle/help/bundle-generate-job/output.txt +++ b/acceptance/bundle/help/bundle-generate-job/output.txt @@ -13,9 +13,10 @@ Flags: -s, --source-dir string Dir path where the downloaded files will be stored (default "src") Global Flags: - --debug enable debug logging - --key string resource key to use for the generated configuration - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + --key string resource key to use for the generated configuration + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle-generate-pipeline/output.txt b/acceptance/bundle/help/bundle-generate-pipeline/output.txt index cf5f70920f..c710ec1266 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/output.txt +++ b/acceptance/bundle/help/bundle-generate-pipeline/output.txt @@ -13,9 +13,10 @@ Flags: -s, --source-dir string Dir path where the downloaded files will be stored (default "src") Global Flags: - --debug enable debug logging - --key string resource key to use for the generated configuration - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + --key string resource key to use for the generated configuration + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 1d77dfdbd4..992a07226e 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -16,10 +16,11 @@ Flags: --key string resource key to use for the generated configuration Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" Use "databricks bundle generate [command] --help" for more information about a command. diff --git a/acceptance/bundle/help/bundle-init/output.txt b/acceptance/bundle/help/bundle-init/output.txt index bafe5a187d..cabc62570c 100644 --- a/acceptance/bundle/help/bundle-init/output.txt +++ b/acceptance/bundle/help/bundle-init/output.txt @@ -24,8 +24,9 @@ Flags: --template-dir string Directory path within a Git repository containing the template. Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle-open/output.txt b/acceptance/bundle/help/bundle-open/output.txt index 8b98aa850c..fff13472cb 100644 --- a/acceptance/bundle/help/bundle-open/output.txt +++ b/acceptance/bundle/help/bundle-open/output.txt @@ -10,8 +10,9 @@ Flags: -h, --help help for open Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index 17763a2953..52eb7a4bb0 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -50,8 +50,9 @@ Flags: --restart Restart the run if it is already running. Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle-schema/output.txt b/acceptance/bundle/help/bundle-schema/output.txt index 8f2983f5b6..71c3bd6183 100644 --- a/acceptance/bundle/help/bundle-schema/output.txt +++ b/acceptance/bundle/help/bundle-schema/output.txt @@ -9,8 +9,9 @@ Flags: -h, --help help for schema Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle-summary/output.txt b/acceptance/bundle/help/bundle-summary/output.txt index 935c4bdc59..c99d41b297 100644 --- a/acceptance/bundle/help/bundle-summary/output.txt +++ b/acceptance/bundle/help/bundle-summary/output.txt @@ -10,8 +10,9 @@ Flags: -h, --help help for summary Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle-sync/output.txt b/acceptance/bundle/help/bundle-sync/output.txt index 6588e6978e..d9f89da6c3 100644 --- a/acceptance/bundle/help/bundle-sync/output.txt +++ b/acceptance/bundle/help/bundle-sync/output.txt @@ -13,7 +13,8 @@ Flags: --watch watch local file system for changes Global Flags: - --debug enable debug logging - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle-validate/output.txt b/acceptance/bundle/help/bundle-validate/output.txt index a0c350faf0..c02157046f 100644 --- a/acceptance/bundle/help/bundle-validate/output.txt +++ b/acceptance/bundle/help/bundle-validate/output.txt @@ -9,8 +9,9 @@ Flags: -h, --help help for validate Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt index e0e2ea47ca..76b24582c5 100644 --- a/acceptance/bundle/help/bundle/output.txt +++ b/acceptance/bundle/help/bundle/output.txt @@ -21,8 +21,9 @@ Available Commands: validate Validate configuration Flags: - -h, --help help for bundle - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + -h, --help help for bundle + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" Global Flags: --debug enable debug logging From 2a1c4d196e314e79e55cf3a139b6fb4a4b4fca5c Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 20 Jan 2025 12:20:37 +0100 Subject: [PATCH 10/29] fix: Message in test --- bundle/config/mutator/set_variables_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go index 07a5c8214f..941925869a 100644 --- a/bundle/config/mutator/set_variables_test.go +++ b/bundle/config/mutator/set_variables_test.go @@ -108,7 +108,7 @@ func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) { require.NoError(t, err) _, err = setVariable(context.Background(), v, &variable, "foo") - assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done through the \"--var\" flag or by setting the BUNDLE_VAR_foo environment variable") + assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done through the \"--var\" or \"--var-file\" flag or by setting the BUNDLE_VAR_foo environment variable") } func TestSetVariablesMutator(t *testing.T) { From bdb8f1e4661cbcbc5e3c900e19623a1c944f2f17 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 20 Jan 2025 12:37:43 +0100 Subject: [PATCH 11/29] fix: Message in acceptance tests --- acceptance/bundle/variables/var_file_overrides/output.txt | 4 ++-- acceptance/bundle/variables/var_file_overrides/script | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt index 0cb274fc65..39417b3802 100644 --- a/acceptance/bundle/variables/var_file_overrides/output.txt +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -112,8 +112,8 @@ Exit code: 1 } } ->>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/empty.json -Error: failed to read variables file: open var_files/empty.json: no such file or directory +>>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/without_required.json +Error: no value assigned to required variable cluster_workers. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_cluster_workers environment variable Exit code: 1 diff --git a/acceptance/bundle/variables/var_file_overrides/script b/acceptance/bundle/variables/var_file_overrides/script index 147ce5acba..242df8e582 100644 --- a/acceptance/bundle/variables/var_file_overrides/script +++ b/acceptance/bundle/variables/var_file_overrides/script @@ -28,4 +28,4 @@ trace errcode $CLI bundle validate -o json --var-file=var_files/complex_to_strin trace errcode $CLI bundle validate -o json --var-file=var_files/string_to_complex.json | jq $cluster_expr # variable is required but it's not provided in the file -trace errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/empty.json | jq $cluster_expr +trace errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/without_required.json | jq $cluster_expr From 19b501898597d6af9cae959d6ea9085f4da61619 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 20 Jan 2025 12:47:17 +0100 Subject: [PATCH 12/29] fix: Flaky test --- acceptance/bundle/variables/var_file_overrides/output.txt | 2 +- .../var_file_overrides/var_files/without_required.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt index 39417b3802..fffaee5177 100644 --- a/acceptance/bundle/variables/var_file_overrides/output.txt +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -113,7 +113,7 @@ Exit code: 1 } >>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/without_required.json -Error: no value assigned to required variable cluster_workers. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_cluster_workers environment variable +Error: no value assigned to required variable cluster. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_cluster environment variable Exit code: 1 diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/without_required.json b/acceptance/bundle/variables/var_file_overrides/var_files/without_required.json index 37e52b0fe0..86166408ee 100644 --- a/acceptance/bundle/variables/var_file_overrides/var_files/without_required.json +++ b/acceptance/bundle/variables/var_file_overrides/var_files/without_required.json @@ -1,3 +1,4 @@ { - "cluster_key": "mlops_stacks-cluster" + "cluster_key": "mlops_stacks-cluster", + "cluster_workers": 2 } From 44aba7e0e05b50411eb45b322d63d8224cfd710f Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 20 Jan 2025 14:39:54 +0100 Subject: [PATCH 13/29] fix: Remove platform specific part from output --- acceptance/bundle/variables/var_file_overrides/output.txt | 2 +- acceptance/bundle/variables/var_file_overrides/script | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt index fffaee5177..c2c73dff9f 100644 --- a/acceptance/bundle/variables/var_file_overrides/output.txt +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -31,7 +31,7 @@ Exit code: 1 } >>> errcode $CLI bundle validate -o json --var-file=var_files/not_found.json -Error: failed to read variables file: open var_files/not_found.json: no such file or directory +Error: failed to read variables file: open var_files/not_found.json: Exit code: 1 diff --git a/acceptance/bundle/variables/var_file_overrides/script b/acceptance/bundle/variables/var_file_overrides/script index 242df8e582..981d1694f1 100644 --- a/acceptance/bundle/variables/var_file_overrides/script +++ b/acceptance/bundle/variables/var_file_overrides/script @@ -10,7 +10,7 @@ trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq trace errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr # file not found -trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json | jq $cluster_expr +trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json 2> >(sed 's/\(Error: failed to read variables file: open var_files\/not_found.json:\).*/\1/' >&2) | jq $cluster_expr # file cannot be parsed trace errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json | jq $cluster_expr From e4164c1776eae1347cc6baa719abbfe3fced1bff Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 20 Jan 2025 16:30:31 +0100 Subject: [PATCH 14/29] fix: Try `jq -b` to preserve window output --- acceptance/bundle/variables/var_file_overrides/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/variables/var_file_overrides/script b/acceptance/bundle/variables/var_file_overrides/script index 981d1694f1..6e51b80dbf 100644 --- a/acceptance/bundle/variables/var_file_overrides/script +++ b/acceptance/bundle/variables/var_file_overrides/script @@ -10,7 +10,7 @@ trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq trace errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr # file not found -trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json 2> >(sed 's/\(Error: failed to read variables file: open var_files\/not_found.json:\).*/\1/' >&2) | jq $cluster_expr +trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json 2> >(sed 's/\(Error: failed to read variables file: open var_files\/not_found.json:\).*/\1/' >&2) | jq -b $cluster_expr # file cannot be parsed trace errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json | jq $cluster_expr From 6dd8fc228e449d4afdf05679a16790a594471903 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 21 Jan 2025 11:39:18 +0100 Subject: [PATCH 15/29] fix: Less flaky sed execution --- acceptance/bundle/variables/var_file_overrides/script | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/variables/var_file_overrides/script b/acceptance/bundle/variables/var_file_overrides/script index 6e51b80dbf..b073409ec0 100644 --- a/acceptance/bundle/variables/var_file_overrides/script +++ b/acceptance/bundle/variables/var_file_overrides/script @@ -10,7 +10,8 @@ trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq trace errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr # file not found -trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json 2> >(sed 's/\(Error: failed to read variables file: open var_files\/not_found.json:\).*/\1/' >&2) | jq -b $cluster_expr +trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json 2> >(sed 's/\(Error: failed to read variables file: open var_files\/not_found.json:\).*/\1/' >&2) > tmp.txt +jq "$cluster_expr" tmp.txt # file cannot be parsed trace errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json | jq $cluster_expr From 1656bf55066f2297e881004c431ed085742070f8 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 21 Jan 2025 14:45:56 +0100 Subject: [PATCH 16/29] fix: Remove redundant test --- .../bundle/variables/var_flag/databricks.yml | 21 ------------------- .../bundle/variables/var_flag/output.txt | 3 --- acceptance/bundle/variables/var_flag/script | 2 -- 3 files changed, 26 deletions(-) delete mode 100644 acceptance/bundle/variables/var_flag/databricks.yml delete mode 100644 acceptance/bundle/variables/var_flag/output.txt delete mode 100644 acceptance/bundle/variables/var_flag/script diff --git a/acceptance/bundle/variables/var_flag/databricks.yml b/acceptance/bundle/variables/var_flag/databricks.yml deleted file mode 100644 index f9de114e53..0000000000 --- a/acceptance/bundle/variables/var_flag/databricks.yml +++ /dev/null @@ -1,21 +0,0 @@ -bundle: - name: TestResolveVariablesFromFlag - -variables: - cluster: - type: "complex" - default: - node_type_id: "undefined" - cluster_key: - default: "undefined" - cluster_workers: - default: 1 - -resources: - jobs: - job1: - job_clusters: - - job_cluster_key: ${var.cluster_key} - new_cluster: - node_type_id: "${var.cluster.node_type_id}" - num_workers: ${var.cluster_workers} diff --git a/acceptance/bundle/variables/var_flag/output.txt b/acceptance/bundle/variables/var_flag/output.txt deleted file mode 100644 index b5390d371c..0000000000 --- a/acceptance/bundle/variables/var_flag/output.txt +++ /dev/null @@ -1,3 +0,0 @@ - ->>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster -"mlops_stacks-cluster" diff --git a/acceptance/bundle/variables/var_flag/script b/acceptance/bundle/variables/var_flag/script deleted file mode 100644 index 66a1217d8d..0000000000 --- a/acceptance/bundle/variables/var_flag/script +++ /dev/null @@ -1,2 +0,0 @@ -# single flag -trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq .resources.jobs.job1.job_clusters[0].job_cluster_key From 422585c2a89661f038c9bb057b5a241fee5d1aa0 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 21 Jan 2025 18:00:39 +0100 Subject: [PATCH 17/29] feat: Default value for variable path --- .../with-default-variable-file/vars.json | 7 +++++ .../variables/var_file_overrides/.gitignore | 1 + .../var_file_overrides/databricks.yml | 3 ++ .../variables/var_file_overrides/output.txt | 29 +++++++++++-------- .../variables/var_file_overrides/script | 11 ++++--- bundle/config/mutator/set_variables.go | 2 +- bundle/config/root.go | 3 ++ bundle/config/variable/variable.go | 14 ++++----- cmd/bundle/utils/utils.go | 27 +++++++++++++---- cmd/bundle/variables.go | 6 +++- 10 files changed, 72 insertions(+), 31 deletions(-) create mode 100644 acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/vars.json create mode 100644 acceptance/bundle/variables/var_file_overrides/.gitignore diff --git a/acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/vars.json b/acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/vars.json new file mode 100644 index 0000000000..035ff13399 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/vars.json @@ -0,0 +1,7 @@ +{ + "cluster": { + "node_type_id": "Standard_DS3_v3" + }, + "cluster_key": "mlops_stacks-cluster-2", + "cluster_workers": 9 +} diff --git a/acceptance/bundle/variables/var_file_overrides/.gitignore b/acceptance/bundle/variables/var_file_overrides/.gitignore new file mode 100644 index 0000000000..bd1711fd16 --- /dev/null +++ b/acceptance/bundle/variables/var_file_overrides/.gitignore @@ -0,0 +1 @@ +!.databricks diff --git a/acceptance/bundle/variables/var_file_overrides/databricks.yml b/acceptance/bundle/variables/var_file_overrides/databricks.yml index cf743a5566..3c8de85517 100644 --- a/acceptance/bundle/variables/var_file_overrides/databricks.yml +++ b/acceptance/bundle/variables/var_file_overrides/databricks.yml @@ -30,3 +30,6 @@ targets: default: "default" without-defaults: + + # see .databricks/bundle/default_target/ for variable values + with-default-variable-file: diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt index c2c73dff9f..f5b9d20b16 100644 --- a/acceptance/bundle/variables/var_file_overrides/output.txt +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -8,25 +8,30 @@ } } ->>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster +>>> $CLI bundle validate -o json --target with-default-variable-file { - "job_cluster_key": "mlops_stacks-cluster", + "job_cluster_key": "mlops_stacks-cluster-2", "new_cluster": { - "node_type_id": "default", - "num_workers": 1 + "node_type_id": "Standard_DS3_v3", + "num_workers": 9 } } ->>> errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var=cluster_key=mlops_stacks-cluster -Error: cannot specify both --var and --var-file flags - +>>> $CLI bundle validate -o json --var-file=var_files/normal.json --var=cluster_key=mlops_stacks-cluster-overriden +{ + "job_cluster_key": "mlops_stacks-cluster-overriden", + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 + } +} -Exit code: 1 +>>> BUNDLE_VAR_cluster_key=incorrectly-overriden $CLI bundle validate -o json --var-file=var_files/normal.json { - "job_cluster_key": "${var.cluster_key}", + "job_cluster_key": "mlops_stacks-cluster", "new_cluster": { - "node_type_id": "${var.cluster.node_type_id}", - "num_workers": "${var.cluster_workers}" + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 } } @@ -113,7 +118,7 @@ Exit code: 1 } >>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/without_required.json -Error: no value assigned to required variable cluster. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_cluster environment variable +Error: no value assigned to required variable cluster. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle//vars.json file Exit code: 1 diff --git a/acceptance/bundle/variables/var_file_overrides/script b/acceptance/bundle/variables/var_file_overrides/script index b073409ec0..9dab54bdac 100644 --- a/acceptance/bundle/variables/var_file_overrides/script +++ b/acceptance/bundle/variables/var_file_overrides/script @@ -3,11 +3,14 @@ cluster_expr=".resources.jobs.job1.job_clusters[0]" # variable file trace $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr -# variable flag -trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr +# default variable file (see .databricks/*) +trace $CLI bundle validate -o json --target with-default-variable-file | jq $cluster_expr -# both variable file and flag -trace errcode $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster" | jq $cluster_expr +# variable file and variable flag +trace $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster-overriden" | jq $cluster_expr + +# variable file and environment variable +trace BUNDLE_VAR_cluster_key=incorrectly-overriden $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr # file not found trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json 2> >(sed 's/\(Error: failed to read variables file: open var_files\/not_found.json:\).*/\1/' >&2) > tmp.txt diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index 7b16571e21..d94f52547f 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -64,7 +64,7 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, } // We should have had a value to set for the variable at this point. - return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" or "--var-file" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) + return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done using "--var" or "--var-file", by setting the %s environment variable, or in .databricks/bundle//vars.json file`, name, bundleVarPrefix+name) } func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diff --git a/bundle/config/root.go b/bundle/config/root.go index 3b3c6e5cec..0bcf9d3b9c 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -269,6 +269,9 @@ func (r *Root) InitializeAnyTypeVariables(vars map[string]any) error { return fmt.Errorf("variable %s has not been defined", name) } + if variable.HasValue() { + continue + } err := variable.Set(val) if err != nil { return fmt.Errorf("failed to assign %s to %s: %s", val, name, err) diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index 76feebaad7..fd2556ed64 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -36,13 +36,13 @@ type Variable struct { // This field stores the resolved value for the variable. The variable are // resolved in the following priority order (from highest to lowest) // - // 1. Command line flag, one of these is used - // a. Variable value obtained from arguments, example: `--var="foo=bar"` - // b. Variable value obtained from the file, example: `--var-file="/path/to/file"` - // 2. Environment variable. eg: BUNDLE_VAR_foo=bar - // 3. Default value as defined in the applicable targets block - // 4. Default value defined in variable definition - // 5. Throw error, since if no default value is defined, then the variable + // 1. Command line flag `--var="foo=bar"` + // 2. Variable value from the file, example: `--var-file="/path/to/file"`. + // If path is not specified the default path is used: ".databricks/bundle//vars.json" + // 3. Environment variable. eg: BUNDLE_VAR_foo=bar + // 4. Default value as defined in the applicable targets block + // 5. Default value defined in variable definition + // 6. Throw error, since if no default value is defined, then the variable // is required Value VariableValue `json:"value,omitempty" bundle:"readonly"` diff --git a/cmd/bundle/utils/utils.go b/cmd/bundle/utils/utils.go index f58ef274ce..e659ae600c 100644 --- a/cmd/bundle/utils/utils.go +++ b/cmd/bundle/utils/utils.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/cmd/root" @@ -13,6 +14,10 @@ import ( "github.com/spf13/cobra" ) +func GetDefaultVariableFilePath(target string) string { + return ".databricks/bundle/" + target + "/vars.json" +} + func configureVariables(cmd *cobra.Command, b *bundle.Bundle, variables []string) diag.Diagnostics { return bundle.ApplyFunc(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err := b.Config.InitializeVariables(variables) @@ -66,17 +71,27 @@ func ConfigureBundleWithVariables(cmd *cobra.Command) (*bundle.Bundle, diag.Diag if err != nil { return b, diag.FromErr(err) } + + if len(variables) > 0 { + // Initialize variables by assigning them values passed as command line flags + diags = diags.Extend(configureVariables(cmd, b, variables)) + } + variableFilePath, err := cmd.Flags().GetString("var-file") if err != nil { return b, diag.FromErr(err) } - if len(variables) > 0 && variableFilePath != "" { - return b, diag.Errorf("cannot specify both --var and --var-file flags") - } else if len(variables) > 0 { - // Initialize variables by assigning them values passed as command line flags - diags = diags.Extend(configureVariables(cmd, b, variables)) - } else if variableFilePath != "" { + if variableFilePath == "" { + // Fallback to default variable file path + defaultPath := GetDefaultVariableFilePath(b.Config.Bundle.Target) + normalisedPath := filepath.Join(b.BundleRootPath, defaultPath) + if _, err := os.Stat(normalisedPath); err == nil { + variableFilePath = normalisedPath + } + } + + if variableFilePath != "" { // Initialize variables by loading them from a file diags = diags.Extend(configureVariablesFromFile(cmd, b, variableFilePath)) } diff --git a/cmd/bundle/variables.go b/cmd/bundle/variables.go index e409e8946c..984d214e40 100644 --- a/cmd/bundle/variables.go +++ b/cmd/bundle/variables.go @@ -1,10 +1,14 @@ package bundle import ( + "fmt" + "github.com/spf13/cobra" + + "github.com/databricks/cli/cmd/bundle/utils" ) func initVariableFlag(cmd *cobra.Command) { cmd.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in bundle config. Example: --var="foo=bar"`) - cmd.PersistentFlags().String("var-file", "", `file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json"`) + cmd.PersistentFlags().String("var-file", "", fmt.Sprintf(`file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default "%s")`, utils.GetDefaultVariableFilePath(""))) } From 47c8fb36518762098ec4104ea06ad26f91eb95f7 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 21 Jan 2025 18:01:58 +0100 Subject: [PATCH 18/29] chore: Regenerate other snapshots --- acceptance/bundle/help/bundle-deploy/output.txt | 2 +- acceptance/bundle/help/bundle-deployment/output.txt | 2 +- acceptance/bundle/help/bundle-destroy/output.txt | 2 +- acceptance/bundle/help/bundle-generate-dashboard/output.txt | 2 +- acceptance/bundle/help/bundle-generate-job/output.txt | 2 +- acceptance/bundle/help/bundle-generate-pipeline/output.txt | 2 +- acceptance/bundle/help/bundle-generate/output.txt | 2 +- acceptance/bundle/help/bundle-init/output.txt | 2 +- acceptance/bundle/help/bundle-open/output.txt | 2 +- acceptance/bundle/help/bundle-run/output.txt | 2 +- acceptance/bundle/help/bundle-schema/output.txt | 2 +- acceptance/bundle/help/bundle-summary/output.txt | 2 +- acceptance/bundle/help/bundle-sync/output.txt | 2 +- acceptance/bundle/help/bundle-validate/output.txt | 2 +- acceptance/bundle/help/bundle/output.txt | 2 +- acceptance/bundle/variables/empty/output.txt | 2 +- acceptance/bundle/variables/env_overrides/output.txt | 2 +- acceptance/bundle/variables/vanilla/output.txt | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/acceptance/bundle/help/bundle-deploy/output.txt b/acceptance/bundle/help/bundle-deploy/output.txt index 0d06436b9c..15f19dd12d 100644 --- a/acceptance/bundle/help/bundle-deploy/output.txt +++ b/acceptance/bundle/help/bundle-deploy/output.txt @@ -19,4 +19,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-deployment/output.txt b/acceptance/bundle/help/bundle-deployment/output.txt index faae384b07..790ff2f1a5 100644 --- a/acceptance/bundle/help/bundle-deployment/output.txt +++ b/acceptance/bundle/help/bundle-deployment/output.txt @@ -18,6 +18,6 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") Use "databricks bundle deployment [command] --help" for more information about a command. diff --git a/acceptance/bundle/help/bundle-destroy/output.txt b/acceptance/bundle/help/bundle-destroy/output.txt index 54fac38b49..0eaeea00be 100644 --- a/acceptance/bundle/help/bundle-destroy/output.txt +++ b/acceptance/bundle/help/bundle-destroy/output.txt @@ -16,4 +16,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-generate-dashboard/output.txt b/acceptance/bundle/help/bundle-generate-dashboard/output.txt index 240a5a580e..6c585a5377 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/output.txt +++ b/acceptance/bundle/help/bundle-generate-dashboard/output.txt @@ -22,4 +22,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-generate-job/output.txt b/acceptance/bundle/help/bundle-generate-job/output.txt index 6bb4fc2c8d..85397ff3bf 100644 --- a/acceptance/bundle/help/bundle-generate-job/output.txt +++ b/acceptance/bundle/help/bundle-generate-job/output.txt @@ -19,4 +19,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-generate-pipeline/output.txt b/acceptance/bundle/help/bundle-generate-pipeline/output.txt index c710ec1266..1a30edc10e 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/output.txt +++ b/acceptance/bundle/help/bundle-generate-pipeline/output.txt @@ -19,4 +19,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 992a07226e..51ae15df6c 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -21,6 +21,6 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") Use "databricks bundle generate [command] --help" for more information about a command. diff --git a/acceptance/bundle/help/bundle-init/output.txt b/acceptance/bundle/help/bundle-init/output.txt index cabc62570c..86164a35a7 100644 --- a/acceptance/bundle/help/bundle-init/output.txt +++ b/acceptance/bundle/help/bundle-init/output.txt @@ -29,4 +29,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-open/output.txt b/acceptance/bundle/help/bundle-open/output.txt index fff13472cb..264349dcc2 100644 --- a/acceptance/bundle/help/bundle-open/output.txt +++ b/acceptance/bundle/help/bundle-open/output.txt @@ -15,4 +15,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index 52eb7a4bb0..f4aba7386e 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -55,4 +55,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-schema/output.txt b/acceptance/bundle/help/bundle-schema/output.txt index 71c3bd6183..198938cea5 100644 --- a/acceptance/bundle/help/bundle-schema/output.txt +++ b/acceptance/bundle/help/bundle-schema/output.txt @@ -14,4 +14,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-summary/output.txt b/acceptance/bundle/help/bundle-summary/output.txt index c99d41b297..af51f3be4c 100644 --- a/acceptance/bundle/help/bundle-summary/output.txt +++ b/acceptance/bundle/help/bundle-summary/output.txt @@ -15,4 +15,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-sync/output.txt b/acceptance/bundle/help/bundle-sync/output.txt index d9f89da6c3..61e9423987 100644 --- a/acceptance/bundle/help/bundle-sync/output.txt +++ b/acceptance/bundle/help/bundle-sync/output.txt @@ -17,4 +17,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-validate/output.txt b/acceptance/bundle/help/bundle-validate/output.txt index c02157046f..633fb1e239 100644 --- a/acceptance/bundle/help/bundle-validate/output.txt +++ b/acceptance/bundle/help/bundle-validate/output.txt @@ -14,4 +14,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt index 76b24582c5..cc98f1800b 100644 --- a/acceptance/bundle/help/bundle/output.txt +++ b/acceptance/bundle/help/bundle/output.txt @@ -23,7 +23,7 @@ Available Commands: Flags: -h, --help help for bundle --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" + --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") Global Flags: --debug enable debug logging diff --git a/acceptance/bundle/variables/empty/output.txt b/acceptance/bundle/variables/empty/output.txt index 589a0a5028..4577d725ab 100644 --- a/acceptance/bundle/variables/empty/output.txt +++ b/acceptance/bundle/variables/empty/output.txt @@ -1,4 +1,4 @@ -Error: no value assigned to required variable a. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_a environment variable +Error: no value assigned to required variable a. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_a environment variable, or in .databricks/bundle//vars.json file Name: empty${var.a} Target: default diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt index faa3505f36..ba78a0159c 100644 --- a/acceptance/bundle/variables/env_overrides/output.txt +++ b/acceptance/bundle/variables/env_overrides/output.txt @@ -9,7 +9,7 @@ "prod-a env-var-b" >>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment -Error: no value assigned to required variable b. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_b environment variable +Error: no value assigned to required variable b. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//vars.json file Name: test bundle Target: env-missing-a-required-variable-assignment diff --git a/acceptance/bundle/variables/vanilla/output.txt b/acceptance/bundle/variables/vanilla/output.txt index 36cf44b5aa..00f47d5c8e 100644 --- a/acceptance/bundle/variables/vanilla/output.txt +++ b/acceptance/bundle/variables/vanilla/output.txt @@ -3,7 +3,7 @@ "abc def" >>> errcode $CLI bundle validate -Error: no value assigned to required variable b. Assignment can be done through the "--var" or "--var-file" flag or by setting the BUNDLE_VAR_b environment variable +Error: no value assigned to required variable b. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//vars.json file Name: ${var.a} ${var.b} Target: default From 6f20e88b4ad49732edbc27fa8e5a2c9bd7b6deb3 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 21 Jan 2025 18:16:06 +0100 Subject: [PATCH 19/29] Added annotations to output --- .../variables/var_file_overrides/output.txt | 11 ++++++++++ .../variables/var_file_overrides/script | 22 +++++++++---------- acceptance/script.prepare | 5 +++++ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt index f5b9d20b16..cc5c98485c 100644 --- a/acceptance/bundle/variables/var_file_overrides/output.txt +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -1,4 +1,5 @@ +=== variable file >>> $CLI bundle validate -o json --var-file=var_files/normal.json { "job_cluster_key": "mlops_stacks-cluster", @@ -8,6 +9,7 @@ } } +=== default variable file (see .databricks/*) >>> $CLI bundle validate -o json --target with-default-variable-file { "job_cluster_key": "mlops_stacks-cluster-2", @@ -17,6 +19,7 @@ } } +=== variable file and variable flag >>> $CLI bundle validate -o json --var-file=var_files/normal.json --var=cluster_key=mlops_stacks-cluster-overriden { "job_cluster_key": "mlops_stacks-cluster-overriden", @@ -26,6 +29,7 @@ } } +=== variable file and environment variable >>> BUNDLE_VAR_cluster_key=incorrectly-overriden $CLI bundle validate -o json --var-file=var_files/normal.json { "job_cluster_key": "mlops_stacks-cluster", @@ -35,6 +39,7 @@ } } +=== file not found >>> errcode $CLI bundle validate -o json --var-file=var_files/not_found.json Error: failed to read variables file: open var_files/not_found.json: @@ -48,6 +53,7 @@ Exit code: 1 } } +=== file cannot be parsed >>> errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json Error: failed to parse variables file: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a') @@ -61,6 +67,7 @@ Exit code: 1 } } +=== file has wrong structure >>> errcode $CLI bundle validate -o json --var-file=var_files/wrong_file_structure.json Error: failed to parse variables file: var_files/wrong_file_structure.json:1:1: expected a map, found a sequence in var_files/wrong_file_structure.json:1:1 @@ -78,6 +85,7 @@ Exit code: 1 } } +=== file has variable name that is not defined >>> errcode $CLI bundle validate -o json --var-file=var_files/undeclared.json Error: variable undeclared_var has not been defined @@ -91,6 +99,7 @@ Exit code: 1 } } +=== file has variable name that is complex but default is string >>> errcode $CLI bundle validate -o json --var-file=var_files/complex_to_string.json Error: expected a map to index "variables.cluster.value.node_type_id", found string @@ -104,6 +113,7 @@ Exit code: 1 } } +=== file has variable name that is string but default is complex >>> errcode $CLI bundle validate -o json --var-file=var_files/string_to_complex.json Error: failed to assign map[node_type_id:Standard_DS3_v2] to cluster_key: variable type is not complex @@ -117,6 +127,7 @@ Exit code: 1 } } +=== variable is required but it's not provided in the file >>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/without_required.json Error: no value assigned to required variable cluster. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle//vars.json file diff --git a/acceptance/bundle/variables/var_file_overrides/script b/acceptance/bundle/variables/var_file_overrides/script index 9dab54bdac..8f1f6700ef 100644 --- a/acceptance/bundle/variables/var_file_overrides/script +++ b/acceptance/bundle/variables/var_file_overrides/script @@ -1,35 +1,35 @@ cluster_expr=".resources.jobs.job1.job_clusters[0]" -# variable file +title "variable file" trace $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr -# default variable file (see .databricks/*) +title "default variable file (see .databricks/*)" trace $CLI bundle validate -o json --target with-default-variable-file | jq $cluster_expr -# variable file and variable flag +title "variable file and variable flag" trace $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster-overriden" | jq $cluster_expr -# variable file and environment variable +title "variable file and environment variable" trace BUNDLE_VAR_cluster_key=incorrectly-overriden $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr -# file not found +title "file not found" trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json 2> >(sed 's/\(Error: failed to read variables file: open var_files\/not_found.json:\).*/\1/' >&2) > tmp.txt jq "$cluster_expr" tmp.txt -# file cannot be parsed +title "file cannot be parsed" trace errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json | jq $cluster_expr -# file has wrong structure +title "file has wrong structure" trace errcode $CLI bundle validate -o json --var-file=var_files/wrong_file_structure.json | jq $cluster_expr -# file has variable name that is not defined +title "file has variable name that is not defined" trace errcode $CLI bundle validate -o json --var-file=var_files/undeclared.json | jq $cluster_expr -# file has variable name that is complex but default is string +title "file has variable name that is complex but default is string" trace errcode $CLI bundle validate -o json --var-file=var_files/complex_to_string.json | jq $cluster_expr -# file has variable name that is string but default is complex +title "file has variable name that is string but default is complex" trace errcode $CLI bundle validate -o json --var-file=var_files/string_to_complex.json | jq $cluster_expr -# variable is required but it's not provided in the file +title "variable is required but it's not provided in the file" trace errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/without_required.json | jq $cluster_expr diff --git a/acceptance/script.prepare b/acceptance/script.prepare index 61061b59e0..4932e45f5c 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -40,3 +40,8 @@ git-repo-init() { git add databricks.yml git commit -qm 'Add databricks.yml' } + +title() { + local label="$1" + printf "\n=== %s" "$label" +} From ad6343a574517cfe666109047af15908582b0d20 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 21 Jan 2025 18:17:06 +0100 Subject: [PATCH 20/29] Cleanup in flag description --- cmd/bundle/variables.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bundle/variables.go b/cmd/bundle/variables.go index 984d214e40..f9f7e45bb2 100644 --- a/cmd/bundle/variables.go +++ b/cmd/bundle/variables.go @@ -10,5 +10,5 @@ import ( func initVariableFlag(cmd *cobra.Command) { cmd.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in bundle config. Example: --var="foo=bar"`) - cmd.PersistentFlags().String("var-file", "", fmt.Sprintf(`file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default "%s")`, utils.GetDefaultVariableFilePath(""))) + cmd.PersistentFlags().String("var-file", "", fmt.Sprintf(`path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default "%s")`, utils.GetDefaultVariableFilePath(""))) } From e0c81f0818241d40f45fac12ddd445698d6255a8 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 21 Jan 2025 18:18:47 +0100 Subject: [PATCH 21/29] Regenerate help snapshots --- acceptance/bundle/help/bundle-deploy/output.txt | 2 +- acceptance/bundle/help/bundle-deployment/output.txt | 2 +- acceptance/bundle/help/bundle-destroy/output.txt | 2 +- acceptance/bundle/help/bundle-generate-dashboard/output.txt | 2 +- acceptance/bundle/help/bundle-generate-job/output.txt | 2 +- acceptance/bundle/help/bundle-generate-pipeline/output.txt | 2 +- acceptance/bundle/help/bundle-generate/output.txt | 2 +- acceptance/bundle/help/bundle-init/output.txt | 2 +- acceptance/bundle/help/bundle-open/output.txt | 2 +- acceptance/bundle/help/bundle-run/output.txt | 2 +- acceptance/bundle/help/bundle-schema/output.txt | 2 +- acceptance/bundle/help/bundle-summary/output.txt | 2 +- acceptance/bundle/help/bundle-sync/output.txt | 2 +- acceptance/bundle/help/bundle-validate/output.txt | 2 +- acceptance/bundle/help/bundle/output.txt | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/acceptance/bundle/help/bundle-deploy/output.txt b/acceptance/bundle/help/bundle-deploy/output.txt index 15f19dd12d..fc2d62ea51 100644 --- a/acceptance/bundle/help/bundle-deploy/output.txt +++ b/acceptance/bundle/help/bundle-deploy/output.txt @@ -19,4 +19,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-deployment/output.txt b/acceptance/bundle/help/bundle-deployment/output.txt index 790ff2f1a5..70d269013e 100644 --- a/acceptance/bundle/help/bundle-deployment/output.txt +++ b/acceptance/bundle/help/bundle-deployment/output.txt @@ -18,6 +18,6 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") Use "databricks bundle deployment [command] --help" for more information about a command. diff --git a/acceptance/bundle/help/bundle-destroy/output.txt b/acceptance/bundle/help/bundle-destroy/output.txt index 0eaeea00be..9c334edba5 100644 --- a/acceptance/bundle/help/bundle-destroy/output.txt +++ b/acceptance/bundle/help/bundle-destroy/output.txt @@ -16,4 +16,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-generate-dashboard/output.txt b/acceptance/bundle/help/bundle-generate-dashboard/output.txt index 6c585a5377..9f3aae3132 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/output.txt +++ b/acceptance/bundle/help/bundle-generate-dashboard/output.txt @@ -22,4 +22,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-generate-job/output.txt b/acceptance/bundle/help/bundle-generate-job/output.txt index 85397ff3bf..66318b3352 100644 --- a/acceptance/bundle/help/bundle-generate-job/output.txt +++ b/acceptance/bundle/help/bundle-generate-job/output.txt @@ -19,4 +19,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-generate-pipeline/output.txt b/acceptance/bundle/help/bundle-generate-pipeline/output.txt index 1a30edc10e..648d013488 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/output.txt +++ b/acceptance/bundle/help/bundle-generate-pipeline/output.txt @@ -19,4 +19,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 51ae15df6c..fc40037248 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -21,6 +21,6 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") Use "databricks bundle generate [command] --help" for more information about a command. diff --git a/acceptance/bundle/help/bundle-init/output.txt b/acceptance/bundle/help/bundle-init/output.txt index 86164a35a7..ad229755d4 100644 --- a/acceptance/bundle/help/bundle-init/output.txt +++ b/acceptance/bundle/help/bundle-init/output.txt @@ -29,4 +29,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-open/output.txt b/acceptance/bundle/help/bundle-open/output.txt index 264349dcc2..a64104b508 100644 --- a/acceptance/bundle/help/bundle-open/output.txt +++ b/acceptance/bundle/help/bundle-open/output.txt @@ -15,4 +15,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index f4aba7386e..76f5e6e6a9 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -55,4 +55,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-schema/output.txt b/acceptance/bundle/help/bundle-schema/output.txt index 198938cea5..d1eeed49e6 100644 --- a/acceptance/bundle/help/bundle-schema/output.txt +++ b/acceptance/bundle/help/bundle-schema/output.txt @@ -14,4 +14,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-summary/output.txt b/acceptance/bundle/help/bundle-summary/output.txt index af51f3be4c..382e99aaca 100644 --- a/acceptance/bundle/help/bundle-summary/output.txt +++ b/acceptance/bundle/help/bundle-summary/output.txt @@ -15,4 +15,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-sync/output.txt b/acceptance/bundle/help/bundle-sync/output.txt index 61e9423987..cb977ba2ca 100644 --- a/acceptance/bundle/help/bundle-sync/output.txt +++ b/acceptance/bundle/help/bundle-sync/output.txt @@ -17,4 +17,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle-validate/output.txt b/acceptance/bundle/help/bundle-validate/output.txt index 633fb1e239..f5971e052b 100644 --- a/acceptance/bundle/help/bundle-validate/output.txt +++ b/acceptance/bundle/help/bundle-validate/output.txt @@ -14,4 +14,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt index cc98f1800b..309556762e 100644 --- a/acceptance/bundle/help/bundle/output.txt +++ b/acceptance/bundle/help/bundle/output.txt @@ -23,7 +23,7 @@ Available Commands: Flags: -h, --help help for bundle --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string file path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") Global Flags: --debug enable debug logging From 38b7d275f7fbe57e7bbf32037231b6948fcf4842 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 21 Jan 2025 18:19:22 +0100 Subject: [PATCH 22/29] Remove unnecessary locations --- acceptance/bundle/variables/var_file_overrides/output.txt | 1 - cmd/bundle/utils/utils.go | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt index cc5c98485c..3037439fe3 100644 --- a/acceptance/bundle/variables/var_file_overrides/output.txt +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -70,7 +70,6 @@ Exit code: 1 === file has wrong structure >>> errcode $CLI bundle validate -o json --var-file=var_files/wrong_file_structure.json Error: failed to parse variables file: var_files/wrong_file_structure.json:1:1: expected a map, found a sequence - in var_files/wrong_file_structure.json:1:1 Variables file must be a JSON object with the following format: {"var1": "value1", "var2": "value2"} diff --git a/cmd/bundle/utils/utils.go b/cmd/bundle/utils/utils.go index e659ae600c..80c7ff7907 100644 --- a/cmd/bundle/utils/utils.go +++ b/cmd/bundle/utils/utils.go @@ -42,10 +42,9 @@ func configureVariablesFromFile(cmd *cobra.Command, b *bundle.Bundle, filePath s err = convert.ToTyped(&vars, val) if err != nil { return diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "failed to parse variables file: " + err.Error(), - Detail: "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}", - Locations: val.Locations(), + Severity: diag.Error, + Summary: "failed to parse variables file: " + err.Error(), + Detail: "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}", }) } From 3e1a3354f0156641fb5293b68174086050fb5229 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Tue, 21 Jan 2025 18:25:26 +0100 Subject: [PATCH 23/29] Fix test with error message --- bundle/config/mutator/set_variables_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go index 941925869a..ece6c83085 100644 --- a/bundle/config/mutator/set_variables_test.go +++ b/bundle/config/mutator/set_variables_test.go @@ -108,7 +108,7 @@ func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) { require.NoError(t, err) _, err = setVariable(context.Background(), v, &variable, "foo") - assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done through the \"--var\" or \"--var-file\" flag or by setting the BUNDLE_VAR_foo environment variable") + assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done using \"--var\" or \"--var-file\", by setting the BUNDLE_VAR_foo environment variable, or in .databricks/bundle//vars.json file") } func TestSetVariablesMutator(t *testing.T) { From 229fdb204d46bc841fb47931cbd053b522e9bed0 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Wed, 22 Jan 2025 13:28:36 +0100 Subject: [PATCH 24/29] Use variable-overrides.json name --- acceptance/bundle/help/bundle-deploy/output.txt | 2 +- acceptance/bundle/help/bundle-deployment/output.txt | 2 +- acceptance/bundle/help/bundle-destroy/output.txt | 2 +- acceptance/bundle/help/bundle-generate-dashboard/output.txt | 2 +- acceptance/bundle/help/bundle-generate-job/output.txt | 2 +- acceptance/bundle/help/bundle-generate-pipeline/output.txt | 2 +- acceptance/bundle/help/bundle-generate/output.txt | 2 +- acceptance/bundle/help/bundle-init/output.txt | 2 +- acceptance/bundle/help/bundle-open/output.txt | 2 +- acceptance/bundle/help/bundle-run/output.txt | 2 +- acceptance/bundle/help/bundle-schema/output.txt | 2 +- acceptance/bundle/help/bundle-summary/output.txt | 2 +- acceptance/bundle/help/bundle-sync/output.txt | 2 +- acceptance/bundle/help/bundle-validate/output.txt | 2 +- acceptance/bundle/help/bundle/output.txt | 2 +- acceptance/bundle/variables/empty/output.txt | 2 +- acceptance/bundle/variables/env_overrides/output.txt | 2 +- acceptance/bundle/variables/vanilla/output.txt | 2 +- .../{vars.json => variable-overrides.json} | 0 acceptance/bundle/variables/var_file_overrides/output.txt | 2 +- bundle/config/mutator/set_variables.go | 2 +- bundle/config/variable/variable.go | 2 +- cmd/bundle/utils/utils.go | 2 +- cmd/bundle/variables.go | 2 +- 24 files changed, 23 insertions(+), 23 deletions(-) rename acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/{vars.json => variable-overrides.json} (100%) diff --git a/acceptance/bundle/help/bundle-deploy/output.txt b/acceptance/bundle/help/bundle-deploy/output.txt index fc2d62ea51..3720183d98 100644 --- a/acceptance/bundle/help/bundle-deploy/output.txt +++ b/acceptance/bundle/help/bundle-deploy/output.txt @@ -19,4 +19,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle-deployment/output.txt b/acceptance/bundle/help/bundle-deployment/output.txt index 70d269013e..9733639223 100644 --- a/acceptance/bundle/help/bundle-deployment/output.txt +++ b/acceptance/bundle/help/bundle-deployment/output.txt @@ -18,6 +18,6 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") Use "databricks bundle deployment [command] --help" for more information about a command. diff --git a/acceptance/bundle/help/bundle-destroy/output.txt b/acceptance/bundle/help/bundle-destroy/output.txt index 9c334edba5..0c4166ebcf 100644 --- a/acceptance/bundle/help/bundle-destroy/output.txt +++ b/acceptance/bundle/help/bundle-destroy/output.txt @@ -16,4 +16,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle-generate-dashboard/output.txt b/acceptance/bundle/help/bundle-generate-dashboard/output.txt index 9f3aae3132..7ffb6b4433 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/output.txt +++ b/acceptance/bundle/help/bundle-generate-dashboard/output.txt @@ -22,4 +22,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle-generate-job/output.txt b/acceptance/bundle/help/bundle-generate-job/output.txt index 66318b3352..a0ccb850aa 100644 --- a/acceptance/bundle/help/bundle-generate-job/output.txt +++ b/acceptance/bundle/help/bundle-generate-job/output.txt @@ -19,4 +19,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle-generate-pipeline/output.txt b/acceptance/bundle/help/bundle-generate-pipeline/output.txt index 648d013488..c04a8c6112 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/output.txt +++ b/acceptance/bundle/help/bundle-generate-pipeline/output.txt @@ -19,4 +19,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index fc40037248..44f4a4cd11 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -21,6 +21,6 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") Use "databricks bundle generate [command] --help" for more information about a command. diff --git a/acceptance/bundle/help/bundle-init/output.txt b/acceptance/bundle/help/bundle-init/output.txt index ad229755d4..e6b7808d81 100644 --- a/acceptance/bundle/help/bundle-init/output.txt +++ b/acceptance/bundle/help/bundle-init/output.txt @@ -29,4 +29,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle-open/output.txt b/acceptance/bundle/help/bundle-open/output.txt index a64104b508..861fd2fc3e 100644 --- a/acceptance/bundle/help/bundle-open/output.txt +++ b/acceptance/bundle/help/bundle-open/output.txt @@ -15,4 +15,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index 76f5e6e6a9..2ea2d282f1 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -55,4 +55,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle-schema/output.txt b/acceptance/bundle/help/bundle-schema/output.txt index d1eeed49e6..394c73e597 100644 --- a/acceptance/bundle/help/bundle-schema/output.txt +++ b/acceptance/bundle/help/bundle-schema/output.txt @@ -14,4 +14,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle-summary/output.txt b/acceptance/bundle/help/bundle-summary/output.txt index 382e99aaca..e49b12927c 100644 --- a/acceptance/bundle/help/bundle-summary/output.txt +++ b/acceptance/bundle/help/bundle-summary/output.txt @@ -15,4 +15,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle-sync/output.txt b/acceptance/bundle/help/bundle-sync/output.txt index cb977ba2ca..b90a32b610 100644 --- a/acceptance/bundle/help/bundle-sync/output.txt +++ b/acceptance/bundle/help/bundle-sync/output.txt @@ -17,4 +17,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle-validate/output.txt b/acceptance/bundle/help/bundle-validate/output.txt index f5971e052b..7ea6c0f5ad 100644 --- a/acceptance/bundle/help/bundle-validate/output.txt +++ b/acceptance/bundle/help/bundle-validate/output.txt @@ -14,4 +14,4 @@ Global Flags: -p, --profile string ~/.databrickscfg profile -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt index 309556762e..aa64f8c798 100644 --- a/acceptance/bundle/help/bundle/output.txt +++ b/acceptance/bundle/help/bundle/output.txt @@ -23,7 +23,7 @@ Available Commands: Flags: -h, --help help for bundle --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default ".databricks/bundle//vars.json") + --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") Global Flags: --debug enable debug logging diff --git a/acceptance/bundle/variables/empty/output.txt b/acceptance/bundle/variables/empty/output.txt index 4577d725ab..cb574ece85 100644 --- a/acceptance/bundle/variables/empty/output.txt +++ b/acceptance/bundle/variables/empty/output.txt @@ -1,4 +1,4 @@ -Error: no value assigned to required variable a. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_a environment variable, or in .databricks/bundle//vars.json file +Error: no value assigned to required variable a. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_a environment variable, or in .databricks/bundle//variable-overrides.json file Name: empty${var.a} Target: default diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt index ba78a0159c..ed16c483e7 100644 --- a/acceptance/bundle/variables/env_overrides/output.txt +++ b/acceptance/bundle/variables/env_overrides/output.txt @@ -9,7 +9,7 @@ "prod-a env-var-b" >>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment -Error: no value assigned to required variable b. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//vars.json file +Error: no value assigned to required variable b. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//variable-overrides.json file Name: test bundle Target: env-missing-a-required-variable-assignment diff --git a/acceptance/bundle/variables/vanilla/output.txt b/acceptance/bundle/variables/vanilla/output.txt index 00f47d5c8e..fa390f9dd4 100644 --- a/acceptance/bundle/variables/vanilla/output.txt +++ b/acceptance/bundle/variables/vanilla/output.txt @@ -3,7 +3,7 @@ "abc def" >>> errcode $CLI bundle validate -Error: no value assigned to required variable b. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//vars.json file +Error: no value assigned to required variable b. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//variable-overrides.json file Name: ${var.a} ${var.b} Target: default diff --git a/acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/vars.json b/acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/variable-overrides.json similarity index 100% rename from acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/vars.json rename to acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/variable-overrides.json diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt index 3037439fe3..988200c817 100644 --- a/acceptance/bundle/variables/var_file_overrides/output.txt +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -128,7 +128,7 @@ Exit code: 1 === variable is required but it's not provided in the file >>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/without_required.json -Error: no value assigned to required variable cluster. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle//vars.json file +Error: no value assigned to required variable cluster. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle//variable-overrides.json file Exit code: 1 diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index d94f52547f..69fb960f8a 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -64,7 +64,7 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, } // We should have had a value to set for the variable at this point. - return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done using "--var" or "--var-file", by setting the %s environment variable, or in .databricks/bundle//vars.json file`, name, bundleVarPrefix+name) + return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done using "--var" or "--var-file", by setting the %s environment variable, or in .databricks/bundle//variable-overrides.json file`, name, bundleVarPrefix+name) } func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index fd2556ed64..a5c3a03076 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -38,7 +38,7 @@ type Variable struct { // // 1. Command line flag `--var="foo=bar"` // 2. Variable value from the file, example: `--var-file="/path/to/file"`. - // If path is not specified the default path is used: ".databricks/bundle//vars.json" + // If path is not specified the default path is used: ".databricks/bundle//variable-overrides.json" // 3. Environment variable. eg: BUNDLE_VAR_foo=bar // 4. Default value as defined in the applicable targets block // 5. Default value defined in variable definition diff --git a/cmd/bundle/utils/utils.go b/cmd/bundle/utils/utils.go index 80c7ff7907..e7419e7925 100644 --- a/cmd/bundle/utils/utils.go +++ b/cmd/bundle/utils/utils.go @@ -15,7 +15,7 @@ import ( ) func GetDefaultVariableFilePath(target string) string { - return ".databricks/bundle/" + target + "/vars.json" + return ".databricks/bundle/" + target + "/variable-overrides.json" } func configureVariables(cmd *cobra.Command, b *bundle.Bundle, variables []string) diag.Diagnostics { diff --git a/cmd/bundle/variables.go b/cmd/bundle/variables.go index f9f7e45bb2..79c7af3cf1 100644 --- a/cmd/bundle/variables.go +++ b/cmd/bundle/variables.go @@ -10,5 +10,5 @@ import ( func initVariableFlag(cmd *cobra.Command) { cmd.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in bundle config. Example: --var="foo=bar"`) - cmd.PersistentFlags().String("var-file", "", fmt.Sprintf(`path to a JSON file containing variables. Example: --var-file="/path/to/vars.json" (default "%s")`, utils.GetDefaultVariableFilePath(""))) + cmd.PersistentFlags().String("var-file", "", fmt.Sprintf(`path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default "%s")`, utils.GetDefaultVariableFilePath(""))) } From 6e8f5f3ace4695c970480b773687bbccd7595e06 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Wed, 22 Jan 2025 13:38:45 +0100 Subject: [PATCH 25/29] Fix message in test --- bundle/config/mutator/set_variables_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go index ece6c83085..de51c1b96d 100644 --- a/bundle/config/mutator/set_variables_test.go +++ b/bundle/config/mutator/set_variables_test.go @@ -108,7 +108,7 @@ func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) { require.NoError(t, err) _, err = setVariable(context.Background(), v, &variable, "foo") - assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done using \"--var\" or \"--var-file\", by setting the BUNDLE_VAR_foo environment variable, or in .databricks/bundle//vars.json file") + assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done using \"--var\" or \"--var-file\", by setting the BUNDLE_VAR_foo environment variable, or in .databricks/bundle//variable-overrides.json file") } func TestSetVariablesMutator(t *testing.T) { From cf42459b34dc6e008efc58ce472c483e0ec63975 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Wed, 22 Jan 2025 15:46:30 +0100 Subject: [PATCH 26/29] Add file path to parsing error --- acceptance/bundle/variables/var_file_overrides/output.txt | 2 +- cmd/bundle/utils/utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt index 988200c817..8f9a617442 100644 --- a/acceptance/bundle/variables/var_file_overrides/output.txt +++ b/acceptance/bundle/variables/var_file_overrides/output.txt @@ -55,7 +55,7 @@ Exit code: 1 === file cannot be parsed >>> errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json -Error: failed to parse variables file: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a') +Error: failed to parse variables file var_files/invalid_json.json: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a') Exit code: 1 diff --git a/cmd/bundle/utils/utils.go b/cmd/bundle/utils/utils.go index e7419e7925..8613f0f478 100644 --- a/cmd/bundle/utils/utils.go +++ b/cmd/bundle/utils/utils.go @@ -35,7 +35,7 @@ func configureVariablesFromFile(cmd *cobra.Command, b *bundle.Bundle, filePath s val, err := jsonloader.LoadJSON(f, filePath) if err != nil { - return diag.FromErr(fmt.Errorf("failed to parse variables file: %w", err)) + return diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err)) } vars := map[string]any{} From d771ce78794d67cdee67e15556ea640eecc6b3df Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 23 Jan 2025 14:05:41 +0100 Subject: [PATCH 27/29] Remove --var-file --- bundle/config/mutator/set_variables.go | 69 +++++++++++++++++++-- bundle/config/mutator/set_variables_test.go | 14 ++--- bundle/config/root.go | 20 ------ bundle/config/variable/variable.go | 5 +- cmd/bundle/utils/utils.go | 68 +------------------- cmd/bundle/variables.go | 5 -- 6 files changed, 76 insertions(+), 105 deletions(-) diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index 69fb960f8a..ac2f660a93 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -3,11 +3,14 @@ package mutator import ( "context" "fmt" + "os" + "path/filepath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/jsonloader" "github.com/databricks/cli/libs/env" ) @@ -23,7 +26,11 @@ func (m *setVariables) Name() string { return "SetVariables" } -func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string) (dyn.Value, error) { +func getDefaultVariableFilePath(target string) string { + return ".databricks/bundle/" + target + "/variable-overrides.json" +} + +func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string, fileDefault dyn.Value) (dyn.Value, error) { // case: variable already has value initialized, so skip if variable.HasValue() { return v, nil @@ -49,6 +56,26 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, return v, nil } + // case: Set the variable to the default value from the variable file + if fileDefault.Kind() != dyn.KindInvalid && fileDefault.Kind() != dyn.KindNil { + hasComplexType := variable.IsComplex() + hasComplexValue := fileDefault.Kind() == dyn.KindMap || fileDefault.Kind() == dyn.KindSequence + + if hasComplexType && !hasComplexValue { + return dyn.InvalidValue, fmt.Errorf(`variable %s is of type complex, but the value in the variable file is not a complex type`, name) + } + if !hasComplexType && hasComplexValue { + return dyn.InvalidValue, fmt.Errorf(`variable %s is not of type complex, but the value in the variable file is a complex type`, name) + } + + v, err := dyn.Set(v, "value", fileDefault) + if err != nil { + return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from variable file to variable %s with error: %v`, name, err) + } + + return v, nil + } + // case: Set the variable to its default value if variable.HasDefault() { vDefault, err := dyn.Get(v, "default") @@ -64,10 +91,43 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, } // We should have had a value to set for the variable at this point. - return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done using "--var" or "--var-file", by setting the %s environment variable, or in .databricks/bundle//variable-overrides.json file`, name, bundleVarPrefix+name) + return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Assignment can be done using "--var", by setting the %s environment variable, or in %s file`, name, bundleVarPrefix+name, getDefaultVariableFilePath("")) +} + +func readVariablesFromFile(b *bundle.Bundle) (dyn.Value, diag.Diagnostics) { + var diags diag.Diagnostics + + filePath := filepath.Join(b.BundleRootPath, getDefaultVariableFilePath(b.Config.Bundle.Target)) + if _, err := os.Stat(filePath); err != nil { + return dyn.InvalidValue, nil + } + + f, err := os.ReadFile(filePath) + if err != nil { + return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to read variables file: %w", err)) + } + + val, err := jsonloader.LoadJSON(f, filePath) + if err != nil { + return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err)) + } + + if val.Kind() != dyn.KindMap { + return dyn.InvalidValue, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("failed to parse variables file %s: invalid format", filePath), + Detail: "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}", + }) + } + + return val, nil } func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + defaults, diags := readVariablesFromFile(b) + if diags.HasError() { + return diags + } err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.Map(v, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { name := p[1].Key() @@ -76,9 +136,10 @@ func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos return dyn.InvalidValue, fmt.Errorf(`variable "%s" is not defined`, name) } - return setVariable(ctx, variable, v, name) + fileDefault, _ := dyn.Get(defaults, name) + return setVariable(ctx, variable, v, name, fileDefault) })) }) - return diag.FromErr(err) + return diags.Extend(diag.FromErr(err)) } diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go index de51c1b96d..d904d5be31 100644 --- a/bundle/config/mutator/set_variables_test.go +++ b/bundle/config/mutator/set_variables_test.go @@ -25,7 +25,7 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - v, err = setVariable(context.Background(), v, &variable, "foo") + v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) require.NoError(t, err) err = convert.ToTyped(&variable, v) @@ -43,7 +43,7 @@ func TestSetVariableUsingDefaultValue(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - v, err = setVariable(context.Background(), v, &variable, "foo") + v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) require.NoError(t, err) err = convert.ToTyped(&variable, v) @@ -65,7 +65,7 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - v, err = setVariable(context.Background(), v, &variable, "foo") + v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) require.NoError(t, err) err = convert.ToTyped(&variable, v) @@ -90,7 +90,7 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - v, err = setVariable(context.Background(), v, &variable, "foo") + v, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) require.NoError(t, err) err = convert.ToTyped(&variable, v) @@ -107,8 +107,8 @@ func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - _, err = setVariable(context.Background(), v, &variable, "foo") - assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done using \"--var\" or \"--var-file\", by setting the BUNDLE_VAR_foo environment variable, or in .databricks/bundle//variable-overrides.json file") + _, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) + assert.ErrorContains(t, err, "no value assigned to required variable foo. Assignment can be done using \"--var\", by setting the BUNDLE_VAR_foo environment variable, or in .databricks/bundle//variable-overrides.json file") } func TestSetVariablesMutator(t *testing.T) { @@ -157,6 +157,6 @@ func TestSetComplexVariablesViaEnvVariablesIsNotAllowed(t *testing.T) { v, err := convert.FromTyped(variable, dyn.NilValue) require.NoError(t, err) - _, err = setVariable(context.Background(), v, &variable, "foo") + _, err = setVariable(context.Background(), v, &variable, "foo", dyn.NilValue) assert.ErrorContains(t, err, "setting via environment variables (BUNDLE_VAR_foo) is not supported for complex variable foo") } diff --git a/bundle/config/root.go b/bundle/config/root.go index 0bcf9d3b9c..b974bcec56 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -260,26 +260,6 @@ func (r *Root) InitializeVariables(vars []string) error { return nil } -// Initializes variables parsed from variable file -// Variables can have any type of value, including complex types -func (r *Root) InitializeAnyTypeVariables(vars map[string]any) error { - for name, val := range vars { - variable, ok := r.Variables[name] - if !ok { - return fmt.Errorf("variable %s has not been defined", name) - } - - if variable.HasValue() { - continue - } - err := variable.Set(val) - if err != nil { - return fmt.Errorf("failed to assign %s to %s: %s", val, name, err) - } - } - return nil -} - func (r *Root) Merge(other *Root) error { // Merge dynamic configuration values. return r.Mutate(func(root dyn.Value) (dyn.Value, error) { diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index a5c3a03076..d7f1cdede9 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -37,9 +37,8 @@ type Variable struct { // resolved in the following priority order (from highest to lowest) // // 1. Command line flag `--var="foo=bar"` - // 2. Variable value from the file, example: `--var-file="/path/to/file"`. - // If path is not specified the default path is used: ".databricks/bundle//variable-overrides.json" - // 3. Environment variable. eg: BUNDLE_VAR_foo=bar + // 2. Environment variable. eg: BUNDLE_VAR_foo=bar + // 3. Load defaults from .databricks/bundle//variable-overrides.json // 4. Default value as defined in the applicable targets block // 5. Default value defined in variable definition // 6. Throw error, since if no default value is defined, then the variable diff --git a/cmd/bundle/utils/utils.go b/cmd/bundle/utils/utils.go index 8613f0f478..ce3774cf55 100644 --- a/cmd/bundle/utils/utils.go +++ b/cmd/bundle/utils/utils.go @@ -2,22 +2,13 @@ package utils import ( "context" - "fmt" - "os" - "path/filepath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn/convert" - "github.com/databricks/cli/libs/dyn/jsonloader" "github.com/spf13/cobra" ) -func GetDefaultVariableFilePath(target string) string { - return ".databricks/bundle/" + target + "/variable-overrides.json" -} - func configureVariables(cmd *cobra.Command, b *bundle.Bundle, variables []string) diag.Diagnostics { return bundle.ApplyFunc(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err := b.Config.InitializeVariables(variables) @@ -25,40 +16,6 @@ func configureVariables(cmd *cobra.Command, b *bundle.Bundle, variables []string }) } -func configureVariablesFromFile(cmd *cobra.Command, b *bundle.Bundle, filePath string) diag.Diagnostics { - var diags diag.Diagnostics - return bundle.ApplyFunc(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - f, err := os.ReadFile(filePath) - if err != nil { - return diag.FromErr(fmt.Errorf("failed to read variables file: %w", err)) - } - - val, err := jsonloader.LoadJSON(f, filePath) - if err != nil { - return diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err)) - } - - vars := map[string]any{} - err = convert.ToTyped(&vars, val) - if err != nil { - return diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "failed to parse variables file: " + err.Error(), - Detail: "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}", - }) - } - - if len(vars) > 0 { - err = b.Config.InitializeAnyTypeVariables(vars) - if err != nil { - return diag.FromErr(err) - } - } - - return nil - }) -} - func ConfigureBundleWithVariables(cmd *cobra.Command) (*bundle.Bundle, diag.Diagnostics) { // Load bundle config and apply target b, diags := root.MustConfigureBundle(cmd) @@ -71,29 +28,8 @@ func ConfigureBundleWithVariables(cmd *cobra.Command) (*bundle.Bundle, diag.Diag return b, diag.FromErr(err) } - if len(variables) > 0 { - // Initialize variables by assigning them values passed as command line flags - diags = diags.Extend(configureVariables(cmd, b, variables)) - } - - variableFilePath, err := cmd.Flags().GetString("var-file") - if err != nil { - return b, diag.FromErr(err) - } - - if variableFilePath == "" { - // Fallback to default variable file path - defaultPath := GetDefaultVariableFilePath(b.Config.Bundle.Target) - normalisedPath := filepath.Join(b.BundleRootPath, defaultPath) - if _, err := os.Stat(normalisedPath); err == nil { - variableFilePath = normalisedPath - } - } - - if variableFilePath != "" { - // Initialize variables by loading them from a file - diags = diags.Extend(configureVariablesFromFile(cmd, b, variableFilePath)) - } + // Initialize variables by assigning them values passed as command line flags + diags = diags.Extend(configureVariables(cmd, b, variables)) return b, diags } diff --git a/cmd/bundle/variables.go b/cmd/bundle/variables.go index 79c7af3cf1..f8f5167ead 100644 --- a/cmd/bundle/variables.go +++ b/cmd/bundle/variables.go @@ -1,14 +1,9 @@ package bundle import ( - "fmt" - "github.com/spf13/cobra" - - "github.com/databricks/cli/cmd/bundle/utils" ) func initVariableFlag(cmd *cobra.Command) { cmd.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in bundle config. Example: --var="foo=bar"`) - cmd.PersistentFlags().String("var-file", "", fmt.Sprintf(`path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default "%s")`, utils.GetDefaultVariableFilePath(""))) } From f1ab4f032390bf2901004e0b350c49f7365e9904 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 23 Jan 2025 14:05:48 +0100 Subject: [PATCH 28/29] Regenerate snapshots --- .../bundle/help/bundle-deploy/output.txt | 11 +- .../bundle/help/bundle-deployment/output.txt | 11 +- .../bundle/help/bundle-destroy/output.txt | 11 +- .../help/bundle-generate-dashboard/output.txt | 13 +- .../help/bundle-generate-job/output.txt | 13 +- .../help/bundle-generate-pipeline/output.txt | 13 +- .../bundle/help/bundle-generate/output.txt | 11 +- acceptance/bundle/help/bundle-init/output.txt | 11 +- acceptance/bundle/help/bundle-open/output.txt | 11 +- acceptance/bundle/help/bundle-run/output.txt | 11 +- .../bundle/help/bundle-schema/output.txt | 11 +- .../bundle/help/bundle-summary/output.txt | 11 +- acceptance/bundle/help/bundle-sync/output.txt | 9 +- .../bundle/help/bundle-validate/output.txt | 11 +- acceptance/bundle/help/bundle/output.txt | 5 +- acceptance/bundle/variables/empty/output.txt | 2 +- .../bundle/variables/env_overrides/output.txt | 2 +- .../variable-overrides.json} | 0 .../bundle/default/variable-overrides.json} | 0 .../invalid_json/variable-overrides.json} | 0 .../variable-overrides.json} | 0 .../without_defaults/variable-overrides.json} | 0 .../variable-overrides.json} | 0 .../.gitignore | 0 .../databricks.yml | 31 +++- .../bundle/variables/file-defaults/output.txt | 72 +++++++++ .../bundle/variables/file-defaults/script | 27 ++++ .../bundle/variables/vanilla/output.txt | 2 +- .../variable-overrides.json | 7 - .../variables/var_file_overrides/output.txt | 141 ------------------ .../variables/var_file_overrides/script | 35 ----- .../var_files/undeclared.json | 3 - 32 files changed, 203 insertions(+), 282 deletions(-) rename acceptance/bundle/variables/{var_file_overrides/var_files/string_to_complex.json => file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json} (100%) rename acceptance/bundle/variables/{var_file_overrides/var_files/normal.json => file-defaults/.databricks/bundle/default/variable-overrides.json} (100%) rename acceptance/bundle/variables/{var_file_overrides/var_files/invalid_json.json => file-defaults/.databricks/bundle/invalid_json/variable-overrides.json} (100%) rename acceptance/bundle/variables/{var_file_overrides/var_files/complex_to_string.json => file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json} (100%) rename acceptance/bundle/variables/{var_file_overrides/var_files/without_required.json => file-defaults/.databricks/bundle/without_defaults/variable-overrides.json} (100%) rename acceptance/bundle/variables/{var_file_overrides/var_files/wrong_file_structure.json => file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json} (100%) rename acceptance/bundle/variables/{var_file_overrides => file-defaults}/.gitignore (100%) rename acceptance/bundle/variables/{var_file_overrides => file-defaults}/databricks.yml (53%) create mode 100644 acceptance/bundle/variables/file-defaults/output.txt create mode 100644 acceptance/bundle/variables/file-defaults/script delete mode 100644 acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/variable-overrides.json delete mode 100644 acceptance/bundle/variables/var_file_overrides/output.txt delete mode 100644 acceptance/bundle/variables/var_file_overrides/script delete mode 100644 acceptance/bundle/variables/var_file_overrides/var_files/undeclared.json diff --git a/acceptance/bundle/help/bundle-deploy/output.txt b/acceptance/bundle/help/bundle-deploy/output.txt index 3720183d98..13c903f3e6 100644 --- a/acceptance/bundle/help/bundle-deploy/output.txt +++ b/acceptance/bundle/help/bundle-deploy/output.txt @@ -14,9 +14,8 @@ Flags: -h, --help help for deploy Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle-deployment/output.txt b/acceptance/bundle/help/bundle-deployment/output.txt index 9733639223..ddf5b33054 100644 --- a/acceptance/bundle/help/bundle-deployment/output.txt +++ b/acceptance/bundle/help/bundle-deployment/output.txt @@ -13,11 +13,10 @@ Flags: -h, --help help for deployment Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" Use "databricks bundle deployment [command] --help" for more information about a command. diff --git a/acceptance/bundle/help/bundle-destroy/output.txt b/acceptance/bundle/help/bundle-destroy/output.txt index 0c4166ebcf..d701643012 100644 --- a/acceptance/bundle/help/bundle-destroy/output.txt +++ b/acceptance/bundle/help/bundle-destroy/output.txt @@ -11,9 +11,8 @@ Flags: -h, --help help for destroy Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle-generate-dashboard/output.txt b/acceptance/bundle/help/bundle-generate-dashboard/output.txt index 7ffb6b4433..a63ce0ff87 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/output.txt +++ b/acceptance/bundle/help/bundle-generate-dashboard/output.txt @@ -16,10 +16,9 @@ Flags: --watch watch for changes to the dashboard and update the configuration Global Flags: - --debug enable debug logging - --key string resource key to use for the generated configuration - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + --key string resource key to use for the generated configuration + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle-generate-job/output.txt b/acceptance/bundle/help/bundle-generate-job/output.txt index a0ccb850aa..adc3f45ae5 100644 --- a/acceptance/bundle/help/bundle-generate-job/output.txt +++ b/acceptance/bundle/help/bundle-generate-job/output.txt @@ -13,10 +13,9 @@ Flags: -s, --source-dir string Dir path where the downloaded files will be stored (default "src") Global Flags: - --debug enable debug logging - --key string resource key to use for the generated configuration - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + --key string resource key to use for the generated configuration + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle-generate-pipeline/output.txt b/acceptance/bundle/help/bundle-generate-pipeline/output.txt index c04a8c6112..cf5f70920f 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/output.txt +++ b/acceptance/bundle/help/bundle-generate-pipeline/output.txt @@ -13,10 +13,9 @@ Flags: -s, --source-dir string Dir path where the downloaded files will be stored (default "src") Global Flags: - --debug enable debug logging - --key string resource key to use for the generated configuration - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + --key string resource key to use for the generated configuration + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 44f4a4cd11..1d77dfdbd4 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -16,11 +16,10 @@ Flags: --key string resource key to use for the generated configuration Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" Use "databricks bundle generate [command] --help" for more information about a command. diff --git a/acceptance/bundle/help/bundle-init/output.txt b/acceptance/bundle/help/bundle-init/output.txt index e6b7808d81..bafe5a187d 100644 --- a/acceptance/bundle/help/bundle-init/output.txt +++ b/acceptance/bundle/help/bundle-init/output.txt @@ -24,9 +24,8 @@ Flags: --template-dir string Directory path within a Git repository containing the template. Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle-open/output.txt b/acceptance/bundle/help/bundle-open/output.txt index 861fd2fc3e..8b98aa850c 100644 --- a/acceptance/bundle/help/bundle-open/output.txt +++ b/acceptance/bundle/help/bundle-open/output.txt @@ -10,9 +10,8 @@ Flags: -h, --help help for open Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index 2ea2d282f1..17763a2953 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -50,9 +50,8 @@ Flags: --restart Restart the run if it is already running. Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle-schema/output.txt b/acceptance/bundle/help/bundle-schema/output.txt index 394c73e597..8f2983f5b6 100644 --- a/acceptance/bundle/help/bundle-schema/output.txt +++ b/acceptance/bundle/help/bundle-schema/output.txt @@ -9,9 +9,8 @@ Flags: -h, --help help for schema Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle-summary/output.txt b/acceptance/bundle/help/bundle-summary/output.txt index e49b12927c..935c4bdc59 100644 --- a/acceptance/bundle/help/bundle-summary/output.txt +++ b/acceptance/bundle/help/bundle-summary/output.txt @@ -10,9 +10,8 @@ Flags: -h, --help help for summary Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle-sync/output.txt b/acceptance/bundle/help/bundle-sync/output.txt index b90a32b610..6588e6978e 100644 --- a/acceptance/bundle/help/bundle-sync/output.txt +++ b/acceptance/bundle/help/bundle-sync/output.txt @@ -13,8 +13,7 @@ Flags: --watch watch local file system for changes Global Flags: - --debug enable debug logging - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle-validate/output.txt b/acceptance/bundle/help/bundle-validate/output.txt index 7ea6c0f5ad..a0c350faf0 100644 --- a/acceptance/bundle/help/bundle-validate/output.txt +++ b/acceptance/bundle/help/bundle-validate/output.txt @@ -9,9 +9,8 @@ Flags: -h, --help help for validate Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt index aa64f8c798..e0e2ea47ca 100644 --- a/acceptance/bundle/help/bundle/output.txt +++ b/acceptance/bundle/help/bundle/output.txt @@ -21,9 +21,8 @@ Available Commands: validate Validate configuration Flags: - -h, --help help for bundle - --var strings set values for variables defined in bundle config. Example: --var="foo=bar" - --var-file string path to a JSON file containing variables. Example: --var-file="/path/to/variable-overrides.json" (default ".databricks/bundle//variable-overrides.json") + -h, --help help for bundle + --var strings set values for variables defined in bundle config. Example: --var="foo=bar" Global Flags: --debug enable debug logging diff --git a/acceptance/bundle/variables/empty/output.txt b/acceptance/bundle/variables/empty/output.txt index cb574ece85..8933443dfa 100644 --- a/acceptance/bundle/variables/empty/output.txt +++ b/acceptance/bundle/variables/empty/output.txt @@ -1,4 +1,4 @@ -Error: no value assigned to required variable a. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_a environment variable, or in .databricks/bundle//variable-overrides.json file +Error: no value assigned to required variable a. Assignment can be done using "--var", by setting the BUNDLE_VAR_a environment variable, or in .databricks/bundle//variable-overrides.json file Name: empty${var.a} Target: default diff --git a/acceptance/bundle/variables/env_overrides/output.txt b/acceptance/bundle/variables/env_overrides/output.txt index ed16c483e7..1ee9ef6256 100644 --- a/acceptance/bundle/variables/env_overrides/output.txt +++ b/acceptance/bundle/variables/env_overrides/output.txt @@ -9,7 +9,7 @@ "prod-a env-var-b" >>> errcode $CLI bundle validate -t env-missing-a-required-variable-assignment -Error: no value assigned to required variable b. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//variable-overrides.json file +Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//variable-overrides.json file Name: test bundle Target: env-missing-a-required-variable-assignment diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/string_to_complex.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json similarity index 100% rename from acceptance/bundle/variables/var_file_overrides/var_files/string_to_complex.json rename to acceptance/bundle/variables/file-defaults/.databricks/bundle/complex_to_string/variable-overrides.json diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/normal.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/default/variable-overrides.json similarity index 100% rename from acceptance/bundle/variables/var_file_overrides/var_files/normal.json rename to acceptance/bundle/variables/file-defaults/.databricks/bundle/default/variable-overrides.json diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/invalid_json.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/invalid_json/variable-overrides.json similarity index 100% rename from acceptance/bundle/variables/var_file_overrides/var_files/invalid_json.json rename to acceptance/bundle/variables/file-defaults/.databricks/bundle/invalid_json/variable-overrides.json diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/complex_to_string.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json similarity index 100% rename from acceptance/bundle/variables/var_file_overrides/var_files/complex_to_string.json rename to acceptance/bundle/variables/file-defaults/.databricks/bundle/string_to_complex/variable-overrides.json diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/without_required.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/without_defaults/variable-overrides.json similarity index 100% rename from acceptance/bundle/variables/var_file_overrides/var_files/without_required.json rename to acceptance/bundle/variables/file-defaults/.databricks/bundle/without_defaults/variable-overrides.json diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/wrong_file_structure.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json similarity index 100% rename from acceptance/bundle/variables/var_file_overrides/var_files/wrong_file_structure.json rename to acceptance/bundle/variables/file-defaults/.databricks/bundle/wrong_file_structure/variable-overrides.json diff --git a/acceptance/bundle/variables/var_file_overrides/.gitignore b/acceptance/bundle/variables/file-defaults/.gitignore similarity index 100% rename from acceptance/bundle/variables/var_file_overrides/.gitignore rename to acceptance/bundle/variables/file-defaults/.gitignore diff --git a/acceptance/bundle/variables/var_file_overrides/databricks.yml b/acceptance/bundle/variables/file-defaults/databricks.yml similarity index 53% rename from acceptance/bundle/variables/var_file_overrides/databricks.yml rename to acceptance/bundle/variables/file-defaults/databricks.yml index 3c8de85517..4df250a361 100644 --- a/acceptance/bundle/variables/var_file_overrides/databricks.yml +++ b/acceptance/bundle/variables/file-defaults/databricks.yml @@ -17,7 +17,7 @@ resources: num_workers: ${var.cluster_workers} targets: - with-defaults: + default: default: true variables: cluster_workers: @@ -29,7 +29,30 @@ targets: cluster_key: default: "default" - without-defaults: + without_defaults: - # see .databricks/bundle/default_target/ for variable values - with-default-variable-file: + complex_to_string: + variables: + cluster_workers: + default: 1 + cluster: + type: "complex" + default: + node_type_id: "default" + cluster_key: + default: "default" + + string_to_complex: + variables: + cluster_workers: + default: 1 + cluster: + type: "complex" + default: + node_type_id: "default" + cluster_key: + default: "default" + + wrong_file_structure: + + invalid_json: diff --git a/acceptance/bundle/variables/file-defaults/output.txt b/acceptance/bundle/variables/file-defaults/output.txt new file mode 100644 index 0000000000..5ad6aafbf8 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/output.txt @@ -0,0 +1,72 @@ + +=== variable file +>>> $CLI bundle validate -o json +{ + "job_cluster_key": "mlops_stacks-cluster", + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 + } +} + +=== variable file and variable flag +>>> $CLI bundle validate -o json --var=cluster_key=mlops_stacks-cluster-overriden +{ + "job_cluster_key": "mlops_stacks-cluster-overriden", + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 + } +} + +=== variable file and environment variable +>>> BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json +{ + "job_cluster_key": "mlops_stacks-cluster-overriden", + "new_cluster": { + "node_type_id": "Standard_DS3_v2", + "num_workers": 2 + } +} + +=== file has variable that is complex but default is string +>>> errcode $CLI bundle validate -o json --target complex_to_string +Error: variable cluster_key is not of type complex, but the value in the variable file is a complex type + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +=== file has variable that is string but default is complex +>>> errcode $CLI bundle validate -o json --target string_to_complex +Error: variable cluster is of type complex, but the value in the variable file is not a complex type + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} + +=== variable is required but it's not provided in the file +>>> errcode $CLI bundle validate -o json --target without_defaults +Error: no value assigned to required variable cluster. Assignment can be done using "--var", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle//variable-overrides.json file + + +Exit code: 1 +{ + "job_cluster_key": "${var.cluster_key}", + "new_cluster": { + "node_type_id": "${var.cluster.node_type_id}", + "num_workers": "${var.cluster_workers}" + } +} diff --git a/acceptance/bundle/variables/file-defaults/script b/acceptance/bundle/variables/file-defaults/script new file mode 100644 index 0000000000..31a0a2ba9a --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/script @@ -0,0 +1,27 @@ +cluster_expr=".resources.jobs.job1.job_clusters[0]" + +# defaults from variable file, see .databricks/bundle//variable-overrides.json + +title "variable file" +trace $CLI bundle validate -o json | jq $cluster_expr + +title "variable file and variable flag" +trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster-overriden" | jq $cluster_expr + +title "variable file and environment variable" +trace BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json | jq $cluster_expr + +# title "file cannot be parsed" +# trace errcode $CLI bundle validate -o json --target invalid_json | jq $cluster_expr + +# title "file has wrong structure" +# trace errcode $CLI bundle validate -o json --target wrong_file_structure | jq $cluster_expr + +title "file has variable that is complex but default is string" +trace errcode $CLI bundle validate -o json --target complex_to_string | jq $cluster_expr + +title "file has variable that is string but default is complex" +trace errcode $CLI bundle validate -o json --target string_to_complex | jq $cluster_expr + +title "variable is required but it's not provided in the file" +trace errcode $CLI bundle validate -o json --target without_defaults | jq $cluster_expr diff --git a/acceptance/bundle/variables/vanilla/output.txt b/acceptance/bundle/variables/vanilla/output.txt index fa390f9dd4..e98882bb01 100644 --- a/acceptance/bundle/variables/vanilla/output.txt +++ b/acceptance/bundle/variables/vanilla/output.txt @@ -3,7 +3,7 @@ "abc def" >>> errcode $CLI bundle validate -Error: no value assigned to required variable b. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//variable-overrides.json file +Error: no value assigned to required variable b. Assignment can be done using "--var", by setting the BUNDLE_VAR_b environment variable, or in .databricks/bundle//variable-overrides.json file Name: ${var.a} ${var.b} Target: default diff --git a/acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/variable-overrides.json b/acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/variable-overrides.json deleted file mode 100644 index 035ff13399..0000000000 --- a/acceptance/bundle/variables/var_file_overrides/.databricks/bundle/with-default-variable-file/variable-overrides.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "cluster": { - "node_type_id": "Standard_DS3_v3" - }, - "cluster_key": "mlops_stacks-cluster-2", - "cluster_workers": 9 -} diff --git a/acceptance/bundle/variables/var_file_overrides/output.txt b/acceptance/bundle/variables/var_file_overrides/output.txt deleted file mode 100644 index 8f9a617442..0000000000 --- a/acceptance/bundle/variables/var_file_overrides/output.txt +++ /dev/null @@ -1,141 +0,0 @@ - -=== variable file ->>> $CLI bundle validate -o json --var-file=var_files/normal.json -{ - "job_cluster_key": "mlops_stacks-cluster", - "new_cluster": { - "node_type_id": "Standard_DS3_v2", - "num_workers": 2 - } -} - -=== default variable file (see .databricks/*) ->>> $CLI bundle validate -o json --target with-default-variable-file -{ - "job_cluster_key": "mlops_stacks-cluster-2", - "new_cluster": { - "node_type_id": "Standard_DS3_v3", - "num_workers": 9 - } -} - -=== variable file and variable flag ->>> $CLI bundle validate -o json --var-file=var_files/normal.json --var=cluster_key=mlops_stacks-cluster-overriden -{ - "job_cluster_key": "mlops_stacks-cluster-overriden", - "new_cluster": { - "node_type_id": "Standard_DS3_v2", - "num_workers": 2 - } -} - -=== variable file and environment variable ->>> BUNDLE_VAR_cluster_key=incorrectly-overriden $CLI bundle validate -o json --var-file=var_files/normal.json -{ - "job_cluster_key": "mlops_stacks-cluster", - "new_cluster": { - "node_type_id": "Standard_DS3_v2", - "num_workers": 2 - } -} - -=== file not found ->>> errcode $CLI bundle validate -o json --var-file=var_files/not_found.json -Error: failed to read variables file: open var_files/not_found.json: - - -Exit code: 1 -{ - "job_cluster_key": "${var.cluster_key}", - "new_cluster": { - "node_type_id": "${var.cluster.node_type_id}", - "num_workers": "${var.cluster_workers}" - } -} - -=== file cannot be parsed ->>> errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json -Error: failed to parse variables file var_files/invalid_json.json: error decoding JSON at :0:0: invalid character 'o' in literal false (expecting 'a') - - -Exit code: 1 -{ - "job_cluster_key": "${var.cluster_key}", - "new_cluster": { - "node_type_id": "${var.cluster.node_type_id}", - "num_workers": "${var.cluster_workers}" - } -} - -=== file has wrong structure ->>> errcode $CLI bundle validate -o json --var-file=var_files/wrong_file_structure.json -Error: failed to parse variables file: var_files/wrong_file_structure.json:1:1: expected a map, found a sequence - -Variables file must be a JSON object with the following format: -{"var1": "value1", "var2": "value2"} - - -Exit code: 1 -{ - "job_cluster_key": "${var.cluster_key}", - "new_cluster": { - "node_type_id": "${var.cluster.node_type_id}", - "num_workers": "${var.cluster_workers}" - } -} - -=== file has variable name that is not defined ->>> errcode $CLI bundle validate -o json --var-file=var_files/undeclared.json -Error: variable undeclared_var has not been defined - - -Exit code: 1 -{ - "job_cluster_key": "${var.cluster_key}", - "new_cluster": { - "node_type_id": "${var.cluster.node_type_id}", - "num_workers": "${var.cluster_workers}" - } -} - -=== file has variable name that is complex but default is string ->>> errcode $CLI bundle validate -o json --var-file=var_files/complex_to_string.json -Error: expected a map to index "variables.cluster.value.node_type_id", found string - - -Exit code: 1 -{ - "job_cluster_key": "${var.cluster_key}", - "new_cluster": { - "node_type_id": "${var.cluster.node_type_id}", - "num_workers": "${var.cluster_workers}" - } -} - -=== file has variable name that is string but default is complex ->>> errcode $CLI bundle validate -o json --var-file=var_files/string_to_complex.json -Error: failed to assign map[node_type_id:Standard_DS3_v2] to cluster_key: variable type is not complex - - -Exit code: 1 -{ - "job_cluster_key": "${var.cluster_key}", - "new_cluster": { - "node_type_id": "${var.cluster.node_type_id}", - "num_workers": "${var.cluster_workers}" - } -} - -=== variable is required but it's not provided in the file ->>> errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/without_required.json -Error: no value assigned to required variable cluster. Assignment can be done using "--var" or "--var-file", by setting the BUNDLE_VAR_cluster environment variable, or in .databricks/bundle//variable-overrides.json file - - -Exit code: 1 -{ - "job_cluster_key": "${var.cluster_key}", - "new_cluster": { - "node_type_id": "${var.cluster.node_type_id}", - "num_workers": "${var.cluster_workers}" - } -} diff --git a/acceptance/bundle/variables/var_file_overrides/script b/acceptance/bundle/variables/var_file_overrides/script deleted file mode 100644 index 8f1f6700ef..0000000000 --- a/acceptance/bundle/variables/var_file_overrides/script +++ /dev/null @@ -1,35 +0,0 @@ -cluster_expr=".resources.jobs.job1.job_clusters[0]" - -title "variable file" -trace $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr - -title "default variable file (see .databricks/*)" -trace $CLI bundle validate -o json --target with-default-variable-file | jq $cluster_expr - -title "variable file and variable flag" -trace $CLI bundle validate -o json --var-file=var_files/normal.json --var="cluster_key=mlops_stacks-cluster-overriden" | jq $cluster_expr - -title "variable file and environment variable" -trace BUNDLE_VAR_cluster_key=incorrectly-overriden $CLI bundle validate -o json --var-file=var_files/normal.json | jq $cluster_expr - -title "file not found" -trace errcode $CLI bundle validate -o json --var-file=var_files/not_found.json 2> >(sed 's/\(Error: failed to read variables file: open var_files\/not_found.json:\).*/\1/' >&2) > tmp.txt -jq "$cluster_expr" tmp.txt - -title "file cannot be parsed" -trace errcode $CLI bundle validate -o json --var-file=var_files/invalid_json.json | jq $cluster_expr - -title "file has wrong structure" -trace errcode $CLI bundle validate -o json --var-file=var_files/wrong_file_structure.json | jq $cluster_expr - -title "file has variable name that is not defined" -trace errcode $CLI bundle validate -o json --var-file=var_files/undeclared.json | jq $cluster_expr - -title "file has variable name that is complex but default is string" -trace errcode $CLI bundle validate -o json --var-file=var_files/complex_to_string.json | jq $cluster_expr - -title "file has variable name that is string but default is complex" -trace errcode $CLI bundle validate -o json --var-file=var_files/string_to_complex.json | jq $cluster_expr - -title "variable is required but it's not provided in the file" -trace errcode $CLI bundle validate -o json --target without-defaults --var-file=var_files/without_required.json | jq $cluster_expr diff --git a/acceptance/bundle/variables/var_file_overrides/var_files/undeclared.json b/acceptance/bundle/variables/var_file_overrides/var_files/undeclared.json deleted file mode 100644 index 2a137d23d4..0000000000 --- a/acceptance/bundle/variables/var_file_overrides/var_files/undeclared.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "undeclared_var": 1 -} From 8e8287b05c54ff77ee104cbe46f54a5900720c63 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 23 Jan 2025 15:07:12 +0100 Subject: [PATCH 29/29] Adds test case with value from target specified --- .../bundle/with_value/variable-overrides.json | 3 ++ .../variables/file-defaults/databricks.yml | 37 ++++++++----------- .../bundle/variables/file-defaults/output.txt | 10 +++++ .../bundle/variables/file-defaults/script | 3 ++ 4 files changed, 32 insertions(+), 21 deletions(-) create mode 100644 acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json diff --git a/acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json b/acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json new file mode 100644 index 0000000000..686d685482 --- /dev/null +++ b/acceptance/bundle/variables/file-defaults/.databricks/bundle/with_value/variable-overrides.json @@ -0,0 +1,3 @@ +{ + "cluster_key": "mlops_stacks-cluster-from-file" +} diff --git a/acceptance/bundle/variables/file-defaults/databricks.yml b/acceptance/bundle/variables/file-defaults/databricks.yml index 4df250a361..5838843e19 100644 --- a/acceptance/bundle/variables/file-defaults/databricks.yml +++ b/acceptance/bundle/variables/file-defaults/databricks.yml @@ -20,39 +20,34 @@ targets: default: default: true variables: - cluster_workers: - default: 1 + cluster_workers: 1 cluster: - type: "complex" - default: - node_type_id: "default" - cluster_key: - default: "default" + node_type_id: "default" + cluster_key: "default" without_defaults: complex_to_string: variables: - cluster_workers: - default: 1 + cluster_workers: 1 cluster: - type: "complex" - default: - node_type_id: "default" - cluster_key: - default: "default" + node_type_id: "default" + cluster_key: "default" string_to_complex: variables: - cluster_workers: - default: 1 + cluster_workers: 1 cluster: - type: "complex" - default: - node_type_id: "default" - cluster_key: - default: "default" + node_type_id: "default" + cluster_key: "default" wrong_file_structure: invalid_json: + + with_value: + variables: + cluster_workers: 1 + cluster: + node_type_id: "default" + cluster_key: cluster_key_value diff --git a/acceptance/bundle/variables/file-defaults/output.txt b/acceptance/bundle/variables/file-defaults/output.txt index 5ad6aafbf8..73830aae3a 100644 --- a/acceptance/bundle/variables/file-defaults/output.txt +++ b/acceptance/bundle/variables/file-defaults/output.txt @@ -29,6 +29,16 @@ } } +=== variable has value in config file +>>> $CLI bundle validate -o json --target with_value +{ + "job_cluster_key": "mlops_stacks-cluster-from-file", + "new_cluster": { + "node_type_id": "default", + "num_workers": 1 + } +} + === file has variable that is complex but default is string >>> errcode $CLI bundle validate -o json --target complex_to_string Error: variable cluster_key is not of type complex, but the value in the variable file is a complex type diff --git a/acceptance/bundle/variables/file-defaults/script b/acceptance/bundle/variables/file-defaults/script index 31a0a2ba9a..c5b208755c 100644 --- a/acceptance/bundle/variables/file-defaults/script +++ b/acceptance/bundle/variables/file-defaults/script @@ -11,6 +11,9 @@ trace $CLI bundle validate -o json --var="cluster_key=mlops_stacks-cluster-overr title "variable file and environment variable" trace BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden $CLI bundle validate -o json | jq $cluster_expr +title "variable has value in config file" +trace $CLI bundle validate -o json --target with_value | jq $cluster_expr + # title "file cannot be parsed" # trace errcode $CLI bundle validate -o json --target invalid_json | jq $cluster_expr