From 31b7ad071c4b1f836a1170e369045f9ac4911f84 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 28 Feb 2025 14:12:15 +0100 Subject: [PATCH 01/77] Add the auth.EnvVars function --- libs/auth/env.go | 14 ++++++++++++++ libs/auth/env_test.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/libs/auth/env.go b/libs/auth/env.go index 5c0d212929..f58f29ef7d 100644 --- a/libs/auth/env.go +++ b/libs/auth/env.go @@ -38,3 +38,17 @@ func GetEnvFor(name string) (string, bool) { return "", false } + +func EnvVars() []string { + out := []string{} + + for _, attr := range config.ConfigAttributes { + if len(attr.EnvVars) == 0 { + continue + } + + out = append(out, attr.EnvVars[0]) + } + + return out +} diff --git a/libs/auth/env_test.go b/libs/auth/env_test.go index 850110b602..d7efd0ddc4 100644 --- a/libs/auth/env_test.go +++ b/libs/auth/env_test.go @@ -79,3 +79,40 @@ func TestGetEnvFor(t *testing.T) { assert.False(t, ok) assert.Empty(t, out) } + +func TestAuthEnvVars(t *testing.T) { + expected := []string{ + "DATABRICKS_HOST", + "DATABRICKS_CLUSTER_ID", + "DATABRICKS_WAREHOUSE_ID", + "DATABRICKS_SERVERLESS_COMPUTE_ID", + "DATABRICKS_METADATA_SERVICE_URL", + "DATABRICKS_ACCOUNT_ID", + "DATABRICKS_TOKEN", + "DATABRICKS_USERNAME", + "DATABRICKS_PASSWORD", + "DATABRICKS_CONFIG_PROFILE", + "DATABRICKS_CONFIG_FILE", + "DATABRICKS_GOOGLE_SERVICE_ACCOUNT", + "GOOGLE_CREDENTIALS", + "DATABRICKS_AZURE_RESOURCE_ID", + "ARM_USE_MSI", + "ARM_CLIENT_SECRET", + "ARM_CLIENT_ID", + "ARM_TENANT_ID", + "ACTIONS_ID_TOKEN_REQUEST_URL", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN", + "ARM_ENVIRONMENT", + "DATABRICKS_AZURE_LOGIN_APP_ID", + "DATABRICKS_CLIENT_ID", + "DATABRICKS_CLIENT_SECRET", + "DATABRICKS_CLI_PATH", + "DATABRICKS_AUTH_TYPE", + "DATABRICKS_DEBUG_TRUNCATE_BYTES", + "DATABRICKS_DEBUG_HEADERS", + "DATABRICKS_RATE_LIMIT", + } + + out := EnvVars() + assert.Equal(t, expected, out) +} From bf234c4f5018bb3096aba920d8e281a5e53c5715 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 28 Feb 2025 14:41:37 +0100 Subject: [PATCH 02/77] Add the auth.ProcessEnv function --- internal/testutil/env.go | 31 ++++++++++++-------- libs/auth/env.go | 61 ++++++++++++++++++++++++++++++++++++++-- libs/auth/env_test.go | 36 +++++++++++++++++++++++- 3 files changed, 114 insertions(+), 14 deletions(-) diff --git a/internal/testutil/env.go b/internal/testutil/env.go index 598229655f..0aa97d7799 100644 --- a/internal/testutil/env.go +++ b/internal/testutil/env.go @@ -13,19 +13,11 @@ import ( // The original environment is restored upon test completion. // Note: use of this function is incompatible with parallel execution. func CleanupEnvironment(t TestingT) { - // Restore environment when test finishes. - environ := os.Environ() - t.Cleanup(func() { - // Restore original environment. - for _, kv := range environ { - kvs := strings.SplitN(kv, "=", 2) - os.Setenv(kvs[0], kvs[1]) - } - }) - path := os.Getenv("PATH") pwd := os.Getenv("PWD") - os.Clearenv() + + // Clear all environment variables. + ClearEnvironment(t) // We use t.Setenv instead of os.Setenv because the former actively // prevents a test being run with t.Parallel. Modifying the environment @@ -38,6 +30,23 @@ func CleanupEnvironment(t TestingT) { } } +// ClearEnvironment sets up an empty environment with no environment variables set. +// The original environment is restored upon test completion. +// Note: use of this function is incompatible with parallel execution +func ClearEnvironment(t TestingT) { + // Restore environment when test finishes. + environ := os.Environ() + t.Cleanup(func() { + // Restore original environment. + for _, kv := range environ { + kvs := strings.SplitN(kv, "=", 2) + os.Setenv(kvs[0], kvs[1]) + } + }) + + os.Clearenv() +} + // Changes into specified directory for the duration of the test. // Returns the current working directory. func Chdir(t TestingT, dir string) string { diff --git a/libs/auth/env.go b/libs/auth/env.go index f58f29ef7d..6b53d22a60 100644 --- a/libs/auth/env.go +++ b/libs/auth/env.go @@ -1,6 +1,14 @@ package auth -import "github.com/databricks/databricks-sdk-go/config" +import ( + "fmt" + "os" + "slices" + "sort" + "strings" + + "github.com/databricks/databricks-sdk-go/config" +) // Env generates the authentication environment variables we need to set for // downstream applications from the CLI to work correctly. @@ -39,7 +47,7 @@ func GetEnvFor(name string) (string, bool) { return "", false } -func EnvVars() []string { +func envVars() []string { out := []string{} for _, attr := range config.ConfigAttributes { @@ -52,3 +60,52 @@ func EnvVars() []string { return out } + +// ProcessEnv generates the environment variables can be set to authenticate downstream +// processes to use the same auth credentials as in cfg. +func ProcessEnv(cfg *config.Config) []string { + // We want child telemetry processes to inherit environment variables like $HOME or $HTTPS_PROXY + // because they influence auth resolution. + base := os.Environ() + + out := []string{} + authEnvVars := envVars() + + // Remove any existing auth environment variables. This is done because + // the CLI offers multiple modalities of configuring authentication like + // `--profile` or `DATABRICKS_CONFIG_PROFILE` or `profile: ` in the + // bundle config file. + // + // Each of these modalities have different priorities and thus we don't want + // any auth configuration to piggyback into the child process environment. + // + // This is a precaution to avoid conflicting auth configurations being passed + // to the child telemetry process. + // + // Normally this should be unnecessary because the SDK should error if multiple + // authentication methods have been configured. But there is no harm in doing this + // as a precaution. + for _, v := range base { + k, _, found := strings.Cut(v, "=") + if !found { + continue + } + if slices.Contains(authEnvVars, k) { + continue + } + out = append(out, v) + } + + // Now add the necessary authentication environment variables. + newEnv := Env(cfg) + for k, v := range newEnv { + out = append(out, fmt.Sprintf("%s=%s", k, v)) + } + + // Sort the environment variables so that the output is deterministic. + sort.Slice(out, func(i, j int) bool { + return out[i] < out[j] + }) + + return out +} diff --git a/libs/auth/env_test.go b/libs/auth/env_test.go index d7efd0ddc4..cf771cce4d 100644 --- a/libs/auth/env_test.go +++ b/libs/auth/env_test.go @@ -3,6 +3,7 @@ package auth import ( "testing" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" ) @@ -113,6 +114,39 @@ func TestAuthEnvVars(t *testing.T) { "DATABRICKS_RATE_LIMIT", } - out := EnvVars() + out := envVars() + assert.Equal(t, expected, out) +} + +func TestAuthProcessEnv(t *testing.T) { + testutil.ClearEnvironment(t) + + // Environment variables that should be inherited by child processes. + t.Setenv("HOME", "/home/user") + t.Setenv("HTTPS_PROXY", "http://proxy.com") + + // Environment variables that should be cleaned up by process env: + t.Setenv("DATABRICKS_HOST", "https://test.com") + t.Setenv("DATABRICKS_TOKEN", "test-token") + t.Setenv("DATABRICKS_PASSWORD", "test-password") + t.Setenv("DATABRICKS_METADATA_SERVICE_URL", "http://somurl.com") + t.Setenv("ARM_USE_MSI", "true") + t.Setenv("ARM_TENANT_ID", "test-tenant-id") + t.Setenv("ARM_CLIENT_ID", "test-client-id") + t.Setenv("ARM_CLIENT_SECRET", "test-client-secret") + + in := &config.Config{ + Host: "https://newhost.com", + Token: "new-token", + } + + expected := []string{ + "DATABRICKS_HOST=https://newhost.com", + "DATABRICKS_TOKEN=new-token", + "HOME=/home/user", + "HTTPS_PROXY=http://proxy.com", + } + + out := ProcessEnv(in) assert.Equal(t, expected, out) } From 42f6ecf6d72ca8cc312ae7ec56adc681c69dab78 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 28 Feb 2025 14:47:16 +0100 Subject: [PATCH 03/77] - --- libs/auth/env_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/auth/env_test.go b/libs/auth/env_test.go index cf771cce4d..e04153ed30 100644 --- a/libs/auth/env_test.go +++ b/libs/auth/env_test.go @@ -125,7 +125,7 @@ func TestAuthProcessEnv(t *testing.T) { t.Setenv("HOME", "/home/user") t.Setenv("HTTPS_PROXY", "http://proxy.com") - // Environment variables that should be cleaned up by process env: + // Environment variables that should be cleaned up by ProcessEnv(): t.Setenv("DATABRICKS_HOST", "https://test.com") t.Setenv("DATABRICKS_TOKEN", "test-token") t.Setenv("DATABRICKS_PASSWORD", "test-password") From 4f43fb9acf8892cc8a95905e057e19ca42deb10d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 2 Mar 2025 18:11:46 +0100 Subject: [PATCH 04/77] [WIP] Add bundle exec --- acceptance/acceptance_test.go | 2 +- acceptance/bundle/exec/basic/output.txt | 5 ++ acceptance/bundle/exec/basic/script | 1 + acceptance/bundle/help/bundle/output.txt | 1 + cmd/bundle/bundle.go | 1 + cmd/bundle/exec.go | 109 +++++++++++++++++++++++ 6 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/exec/basic/output.txt create mode 100644 acceptance/bundle/exec/basic/script create mode 100644 cmd/bundle/exec.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 066a84299e..f8b259643a 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -41,7 +41,7 @@ var ( // In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty" // Then install your breakpoints and click "debug test" near TestAccept in VSCODE. // example: var SingleTest = "bundle/variables/empty" -var SingleTest = "" +var SingleTest = "bundle/exec/basic" // If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs // CLI commands. The $CLI in test scripts is a helper that just forwards command-line arguments to this server (see bin/callserver.py). diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt new file mode 100644 index 0000000000..bc6165d261 --- /dev/null +++ b/acceptance/bundle/exec/basic/output.txt @@ -0,0 +1,5 @@ + +>>> errcode [CLI] bundle exec -- echo Hello, World! +Error: Please add a '--' separator. Usage: 'databricks bundle exec -- arg1 arg2 ...' + +Exit code: 1 diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script new file mode 100644 index 0000000000..f0d714b9a4 --- /dev/null +++ b/acceptance/bundle/exec/basic/script @@ -0,0 +1 @@ +trace errcode $CLI bundle exec echo "Hello,\ World" diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt index fc6dd623dd..bb885c80e5 100644 --- a/acceptance/bundle/help/bundle/output.txt +++ b/acceptance/bundle/help/bundle/output.txt @@ -11,6 +11,7 @@ Available Commands: deploy Deploy bundle deployment Deployment related commands destroy Destroy deployed bundle resources + exec Execute a command using the same authentication context as the bundle generate Generate bundle configuration init Initialize using a bundle template open Open a resource in the browser diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index fb88cd7d05..e0818c2f97 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -28,5 +28,6 @@ func New() *cobra.Command { cmd.AddCommand(newDebugCommand()) cmd.AddCommand(deployment.NewDeploymentCommand()) cmd.AddCommand(newOpenCommand()) + cmd.AddCommand(newExecCommand()) return cmd } diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go new file mode 100644 index 0000000000..efc8164f59 --- /dev/null +++ b/cmd/bundle/exec.go @@ -0,0 +1,109 @@ +package bundle + +import ( + "bufio" + "fmt" + "os/exec" + "strings" + "sync" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/auth" + "github.com/spf13/cobra" +) + +// TODO: Confirm that quoted strings are parsed as a single argument. +// TODO: test that -- works with flags as well. +// TODO CONTINUE: Making the bundle exec function work. +// TODO CONTINUE: Adding the scripts section to DABs. + +func newExecCommand() *cobra.Command { + variadicArgs := []string{} + + execCmd := &cobra.Command{ + Use: "exec", + Short: "Execute a command using the same authentication context as the bundle", + Args: cobra.MinimumNArgs(1), + Long: `Examples: + // TODO: Ensure that these multi work strings work with the exec command. +1. databricks bundle exec -- echo "Hello, world!" +2. databricks bundle exec -- /bin/bash -c "echo 'Hello, world!'" +3. databricks bundle exec -- uv run pytest"`, + RunE: func(cmd *cobra.Command, args []string) error { + if cmd.ArgsLenAtDash() != 0 { + return fmt.Errorf("Please add a '--' separator. Usage: 'databricks bundle exec -- %s'", strings.Join(args, " ")) + } + + // Load the bundle configuration to get the authentication credentials + // set in the context. + // TODO: What happens when no bundle is configured? + _, diags := root.MustConfigureBundle(cmd) + if diags.HasError() { + return diags.Error() + } + + childCmd := exec.Command(args[1], args[2:]...) + childCmd.Env = auth.ProcessEnv(root.ConfigUsed(cmd.Context())) + + // Create pipes for stdout and stderr + stdout, err := childCmd.StdoutPipe() + if err != nil { + return fmt.Errorf("Error creating stdout pipe: %w", err) + } + + stderr, err := childCmd.StderrPipe() + if err != nil { + return fmt.Errorf("Error creating stderr pipe: %w", err) + } + + // Start the command + if err := childCmd.Start(); err != nil { + return fmt.Errorf("Error starting command: %s\n", err) + } + + // Stream both stdout and stderr to the user. + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + fmt.Println(scanner.Text()) + } + }() + + go func() { + defer wg.Done() + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + fmt.Println(scanner.Text()) + } + }() + + // Wait for the command to finish. + // TODO: Pretty exit codes? + // TODO: Make CLI return the same exit codes? + err = childCmd.Wait() + if exitErr, ok := err.(*exec.ExitError); ok { + return fmt.Errorf("Command exited with code: %d", exitErr.ExitCode()) + } + + if err := childCmd.Wait(); err != nil { + return fmt.Errorf("Error waiting for command: %w", err) + } + + // Wait for the goroutines to finish printing to stdout and stderr. + wg.Wait() + + return nil + }, + } + + // TODO: Is this needed to make -- work with flags? + // execCmd.Flags().SetInterspersed(false) + + // TODO: func (c *Command) ArgsLenAtDash() int solves my problems here. + + return execCmd +} From 90c6ad0fdb8f84d96ca3853d0156b8f32ff6d541 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 10:52:31 +0100 Subject: [PATCH 05/77] - --- cmd/bundle/exec.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index efc8164f59..11b38ef18e 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -16,16 +16,15 @@ import ( // TODO: test that -- works with flags as well. // TODO CONTINUE: Making the bundle exec function work. // TODO CONTINUE: Adding the scripts section to DABs. +// TODO: Ensure that these multi word strings work with the exec command. Example: echo "Hello, world!" +// Or if it does not work, be sure why. Probably because string parsing is a part of the bash shell. func newExecCommand() *cobra.Command { - variadicArgs := []string{} - execCmd := &cobra.Command{ Use: "exec", Short: "Execute a command using the same authentication context as the bundle", Args: cobra.MinimumNArgs(1), Long: `Examples: - // TODO: Ensure that these multi work strings work with the exec command. 1. databricks bundle exec -- echo "Hello, world!" 2. databricks bundle exec -- /bin/bash -c "echo 'Hello, world!'" 3. databricks bundle exec -- uv run pytest"`, From 6002e0d0402525e3c265fbb44f177ae91ef245bb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 14:01:41 +0100 Subject: [PATCH 06/77] some more tests --- acceptance/acceptance_test.go | 2 +- acceptance/bundle/exec/basic/databricks.yml | 2 ++ acceptance/bundle/exec/basic/output.txt | 5 +++-- acceptance/bundle/exec/basic/script | 2 +- acceptance/bundle/exec/no-separator/output.txt | 5 +++++ acceptance/bundle/exec/no-separator/script | 1 + cmd/bundle/exec.go | 8 ++++---- 7 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 acceptance/bundle/exec/basic/databricks.yml create mode 100644 acceptance/bundle/exec/no-separator/output.txt create mode 100644 acceptance/bundle/exec/no-separator/script diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index f8b259643a..066a84299e 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -41,7 +41,7 @@ var ( // In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty" // Then install your breakpoints and click "debug test" near TestAccept in VSCODE. // example: var SingleTest = "bundle/variables/empty" -var SingleTest = "bundle/exec/basic" +var SingleTest = "" // If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs // CLI commands. The $CLI in test scripts is a helper that just forwards command-line arguments to this server (see bin/callserver.py). diff --git a/acceptance/bundle/exec/basic/databricks.yml b/acceptance/bundle/exec/basic/databricks.yml new file mode 100644 index 0000000000..432311dab0 --- /dev/null +++ b/acceptance/bundle/exec/basic/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index bc6165d261..d45ebfb839 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -1,5 +1,6 @@ ->>> errcode [CLI] bundle exec -- echo Hello, World! -Error: Please add a '--' separator. Usage: 'databricks bundle exec -- arg1 arg2 ...' +>>> errcode [CLI] bundle exec -- echo hello +hello +Error: Error waiting for command: exec: Wait was already called Exit code: 1 diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index f0d714b9a4..c65cd19eab 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -1 +1 @@ -trace errcode $CLI bundle exec echo "Hello,\ World" +trace errcode $CLI bundle exec -- echo hello diff --git a/acceptance/bundle/exec/no-separator/output.txt b/acceptance/bundle/exec/no-separator/output.txt new file mode 100644 index 0000000000..ef7b6f1ce1 --- /dev/null +++ b/acceptance/bundle/exec/no-separator/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle exec echo hello +Error: Please add a '--' separator. Usage: 'databricks bundle exec -- echo hello' + +Exit code: 1 diff --git a/acceptance/bundle/exec/no-separator/script b/acceptance/bundle/exec/no-separator/script new file mode 100644 index 0000000000..050538fa4d --- /dev/null +++ b/acceptance/bundle/exec/no-separator/script @@ -0,0 +1 @@ +trace $CLI bundle exec echo hello diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 11b38ef18e..daeb0c4a2b 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -25,8 +25,8 @@ func newExecCommand() *cobra.Command { Short: "Execute a command using the same authentication context as the bundle", Args: cobra.MinimumNArgs(1), Long: `Examples: -1. databricks bundle exec -- echo "Hello, world!" -2. databricks bundle exec -- /bin/bash -c "echo 'Hello, world!'" +1. databricks bundle exec -- echo hello +2. databricks bundle exec -- /bin/bash -c "echo hello"" 3. databricks bundle exec -- uv run pytest"`, RunE: func(cmd *cobra.Command, args []string) error { if cmd.ArgsLenAtDash() != 0 { @@ -41,7 +41,7 @@ func newExecCommand() *cobra.Command { return diags.Error() } - childCmd := exec.Command(args[1], args[2:]...) + childCmd := exec.Command(args[0], args[1:]...) childCmd.Env = auth.ProcessEnv(root.ConfigUsed(cmd.Context())) // Create pipes for stdout and stderr @@ -82,7 +82,7 @@ func newExecCommand() *cobra.Command { // Wait for the command to finish. // TODO: Pretty exit codes? - // TODO: Make CLI return the same exit codes? + // TODO: Make CLI return the same exit codes? It has to, that's a requirement. err = childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { return fmt.Errorf("Command exited with code: %d", exitErr.ExitCode()) From 1b5ff4887315db3a2334a82646f8a9024b8085f9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 15:12:50 +0100 Subject: [PATCH 07/77] execute scripts from bundle root --- .vscode/settings.json | 5 ++++- acceptance/bundle/exec/basic/output.txt | 8 ++++---- acceptance/bundle/exec/basic/script | 4 +++- acceptance/bundle/exec/cwd/a/b/c/.gitkeep | 0 acceptance/bundle/exec/cwd/databricks.yml | 2 ++ acceptance/bundle/exec/cwd/output.txt | 8 ++++++++ acceptance/bundle/exec/cwd/script | 6 ++++++ acceptance/bundle/exec/no-auth/databricks.yml | 2 ++ acceptance/bundle/exec/no-auth/output.txt | 6 ++++++ acceptance/bundle/exec/no-auth/script | 8 ++++++++ acceptance/bundle/exec/no-bundle/output.txt | 5 +++++ acceptance/bundle/exec/no-bundle/script | 1 + cmd/bundle/exec.go | 19 ++++++++++++++----- 13 files changed, 63 insertions(+), 11 deletions(-) create mode 100644 acceptance/bundle/exec/cwd/a/b/c/.gitkeep create mode 100644 acceptance/bundle/exec/cwd/databricks.yml create mode 100644 acceptance/bundle/exec/cwd/output.txt create mode 100644 acceptance/bundle/exec/cwd/script create mode 100644 acceptance/bundle/exec/no-auth/databricks.yml create mode 100644 acceptance/bundle/exec/no-auth/output.txt create mode 100644 acceptance/bundle/exec/no-auth/script create mode 100644 acceptance/bundle/exec/no-bundle/output.txt create mode 100644 acceptance/bundle/exec/no-bundle/script diff --git a/.vscode/settings.json b/.vscode/settings.json index f8b04f1269..f103538b7f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,5 +17,8 @@ "python.envFile": "${workspaceRoot}/.env", "python.analysis.stubPath": ".vscode", "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------" + "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", + "files.associations": { + "script": "shellscript" + } } diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index d45ebfb839..c1be99c9e6 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -1,6 +1,6 @@ ->>> errcode [CLI] bundle exec -- echo hello -hello -Error: Error waiting for command: exec: Wait was already called +>>> [CLI] bundle exec -- echo hello, world +hello, world -Exit code: 1 +>>> [CLI] bundle exec -- pwd +[TMPDIR] diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index c65cd19eab..48d20c1bd9 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -1 +1,3 @@ -trace errcode $CLI bundle exec -- echo hello +trace $CLI bundle exec -- echo "hello, world" + +trace $CLI bundle exec -- pwd diff --git a/acceptance/bundle/exec/cwd/a/b/c/.gitkeep b/acceptance/bundle/exec/cwd/a/b/c/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/exec/cwd/databricks.yml b/acceptance/bundle/exec/cwd/databricks.yml new file mode 100644 index 0000000000..432311dab0 --- /dev/null +++ b/acceptance/bundle/exec/cwd/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/exec/cwd/output.txt new file mode 100644 index 0000000000..e6496afab8 --- /dev/null +++ b/acceptance/bundle/exec/cwd/output.txt @@ -0,0 +1,8 @@ + +>>> cd a/b/c + +>>> pwd +[TMPDIR]/a/b/c + +>>> [CLI] bundle exec -- pwd +[TMPDIR] diff --git a/acceptance/bundle/exec/cwd/script b/acceptance/bundle/exec/cwd/script new file mode 100644 index 0000000000..73a0966d3f --- /dev/null +++ b/acceptance/bundle/exec/cwd/script @@ -0,0 +1,6 @@ +trace cd a/b/c + +trace pwd + +# Scripts that bundle exec executes should run from the bundle root. +trace $CLI bundle exec -- pwd diff --git a/acceptance/bundle/exec/no-auth/databricks.yml b/acceptance/bundle/exec/no-auth/databricks.yml new file mode 100644 index 0000000000..432311dab0 --- /dev/null +++ b/acceptance/bundle/exec/no-auth/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/no-auth/output.txt b/acceptance/bundle/exec/no-auth/output.txt new file mode 100644 index 0000000000..fce8e3f613 --- /dev/null +++ b/acceptance/bundle/exec/no-auth/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle exec -- echo hello +hello + +>>> [CLI] bundle exec -- pwd +[TMPDIR] diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/exec/no-auth/script new file mode 100644 index 0000000000..93715d4481 --- /dev/null +++ b/acceptance/bundle/exec/no-auth/script @@ -0,0 +1,8 @@ +export DATABRICKS_HOST="" +export DATABRICKS_TOKEN="" + +# Confirm that bundle exec works for commands that do not require authentication, +# even if authentication is not provided. +trace $CLI bundle exec -- echo hello + +trace $CLI bundle exec -- pwd diff --git a/acceptance/bundle/exec/no-bundle/output.txt b/acceptance/bundle/exec/no-bundle/output.txt new file mode 100644 index 0000000000..7030c5405c --- /dev/null +++ b/acceptance/bundle/exec/no-bundle/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle exec -- echo hello +Error: unable to locate bundle root: databricks.yml not found + +Exit code: 1 diff --git a/acceptance/bundle/exec/no-bundle/script b/acceptance/bundle/exec/no-bundle/script new file mode 100644 index 0000000000..b872f65db1 --- /dev/null +++ b/acceptance/bundle/exec/no-bundle/script @@ -0,0 +1 @@ +trace $CLI bundle exec -- echo hello diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index daeb0c4a2b..64bc70da57 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -24,7 +24,13 @@ func newExecCommand() *cobra.Command { Use: "exec", Short: "Execute a command using the same authentication context as the bundle", Args: cobra.MinimumNArgs(1), - Long: `Examples: + // TODO: format once we have all the documentation here. + // TODO: implement and pass the cwd environment variable. Maybe we can defer + // it until we have a scripts section. + Long: ` +Note: This command executes scripts + +Examples: 1. databricks bundle exec -- echo hello 2. databricks bundle exec -- /bin/bash -c "echo hello"" 3. databricks bundle exec -- uv run pytest"`, @@ -36,7 +42,7 @@ func newExecCommand() *cobra.Command { // Load the bundle configuration to get the authentication credentials // set in the context. // TODO: What happens when no bundle is configured? - _, diags := root.MustConfigureBundle(cmd) + b, diags := root.MustConfigureBundle(cmd) if diags.HasError() { return diags.Error() } @@ -44,7 +50,11 @@ func newExecCommand() *cobra.Command { childCmd := exec.Command(args[0], args[1:]...) childCmd.Env = auth.ProcessEnv(root.ConfigUsed(cmd.Context())) - // Create pipes for stdout and stderr + // Execute all scripts from the bundle root directory. + childCmd.Dir = b.BundleRootPath + + // Create pipes for stdout and stderr. + // TODO: Test streaming of this? Is there a way? stdout, err := childCmd.StdoutPipe() if err != nil { return fmt.Errorf("Error creating stdout pipe: %w", err) @@ -87,8 +97,7 @@ func newExecCommand() *cobra.Command { if exitErr, ok := err.(*exec.ExitError); ok { return fmt.Errorf("Command exited with code: %d", exitErr.ExitCode()) } - - if err := childCmd.Wait(); err != nil { + if err != nil { return fmt.Errorf("Error waiting for command: %w", err) } From c442378f45eea4fb95fa0faecf585122b1af3645 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 15:19:18 +0100 Subject: [PATCH 08/77] clarify the cwd plan --- cmd/bundle/exec.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 64bc70da57..bb6ecfc118 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -12,12 +12,7 @@ import ( "github.com/spf13/cobra" ) -// TODO: Confirm that quoted strings are parsed as a single argument. // TODO: test that -- works with flags as well. -// TODO CONTINUE: Making the bundle exec function work. -// TODO CONTINUE: Adding the scripts section to DABs. -// TODO: Ensure that these multi word strings work with the exec command. Example: echo "Hello, world!" -// Or if it does not work, be sure why. Probably because string parsing is a part of the bash shell. func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ @@ -25,8 +20,6 @@ func newExecCommand() *cobra.Command { Short: "Execute a command using the same authentication context as the bundle", Args: cobra.MinimumNArgs(1), // TODO: format once we have all the documentation here. - // TODO: implement and pass the cwd environment variable. Maybe we can defer - // it until we have a scripts section. Long: ` Note: This command executes scripts @@ -41,16 +34,24 @@ Examples: // Load the bundle configuration to get the authentication credentials // set in the context. - // TODO: What happens when no bundle is configured? b, diags := root.MustConfigureBundle(cmd) if diags.HasError() { return diags.Error() } childCmd := exec.Command(args[0], args[1:]...) + childCmd.Env = auth.ProcessEnv(root.ConfigUsed(cmd.Context())) - // Execute all scripts from the bundle root directory. + // Execute all scripts from the bundle root directory. This behavior can + // be surprising in isolation, but we do it to keep the behavior consistent + // for both cases: + // 1. One shot commands like `databricks bundle exec -- echo hello` + // 2. Scripts that are defined in the scripts section of the DAB. + // + // TODO(shreyas): Add a DATABRICKS_BUNDLE_INITIAL_CWD environment variable + // that users can read to figure out the original CWD. I'll do that when + // adding support for the scripts section. childCmd.Dir = b.BundleRootPath // Create pipes for stdout and stderr. @@ -108,10 +109,8 @@ Examples: }, } - // TODO: Is this needed to make -- work with flags? + // TODO: Is this needed to make -- work with flags? What does this option do? // execCmd.Flags().SetInterspersed(false) - // TODO: func (c *Command) ArgsLenAtDash() int solves my problems here. - return execCmd } From 152d982c9bcd6c5254a7ab70e6a2c91f8caa5c65 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 16:58:57 +0100 Subject: [PATCH 09/77] add cases for the target flag --- .../exec/databricks-cli/basic/databricks.yml | 10 +++++ .../databricks-cli/basic/out.requests.txt | 41 +++++++++++++++++++ .../exec/databricks-cli/basic/output.txt | 24 +++++++++++ .../bundle/exec/databricks-cli/basic/script | 15 +++++++ .../bundle/exec/databricks-cli/test.toml | 2 + cmd/bundle/exec.go | 29 ++++++++++++- 6 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/exec/databricks-cli/basic/databricks.yml create mode 100644 acceptance/bundle/exec/databricks-cli/basic/out.requests.txt create mode 100644 acceptance/bundle/exec/databricks-cli/basic/output.txt create mode 100644 acceptance/bundle/exec/databricks-cli/basic/script create mode 100644 acceptance/bundle/exec/databricks-cli/test.toml diff --git a/acceptance/bundle/exec/databricks-cli/basic/databricks.yml b/acceptance/bundle/exec/databricks-cli/basic/databricks.yml new file mode 100644 index 0000000000..6b782c3257 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/basic/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: foobar + +targets: + pat: + default: true + + oauth: + workspace: + client_id: client_id diff --git a/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt b/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt new file mode 100644 index 0000000000..10a364e97a --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt @@ -0,0 +1,41 @@ +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "method": "GET", + "path": "/oidc/.well-known/oauth-authorization-server" +} +{ + "headers": { + "Authorization": [ + "Basic Y2xpZW50X2lkOmNsaWVudC1zZWNyZXQ=" + ] + }, + "method": "POST", + "path": "/oidc/v1/token", + "raw_body": "grant_type=client_credentials\u0026scope=all-apis" +} +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/bundle/exec/databricks-cli/basic/output.txt b/acceptance/bundle/exec/databricks-cli/basic/output.txt new file mode 100644 index 0000000000..48fcf8a619 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/basic/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] bundle exec -- databricks current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} + +>>> [CLI] bundle exec -t pat -- databricks current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} + +>>> errcode [CLI] bundle exec -t pat -- databricks current-user me -t oauth +Error: cannot resolve bundle auth configuration: validate: more than one authorization method configured: oauth and pat. Config: host=[DATABRICKS_URL], token=***, client_id=client_id, databricks_cli_path=[CLI]. Env: DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_CLI_PATH +Error: Command exited with code: 1 + +Exit code: 1 + +>>> [CLI] bundle exec -t oauth -- databricks current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/bundle/exec/databricks-cli/basic/script b/acceptance/bundle/exec/databricks-cli/basic/script new file mode 100644 index 0000000000..3a3c5efe98 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/basic/script @@ -0,0 +1,15 @@ +# Default target +trace $CLI bundle exec -- databricks current-user me + +# Explicitly select default target +trace $CLI bundle exec -t pat -- databricks current-user me + +# Conflicting targets selected. This should fail because for the child command +# pat would be configured via environment variables and oauth via the CLI resulting +# in more than one authorization method configured. +trace errcode $CLI bundle exec -t pat -- databricks current-user me -t oauth + +# Explicitly select oauth target +export DATABRICKS_TOKEN="" +export DATABRICKS_CLIENT_SECRET="client-secret" +trace $CLI bundle exec -t oauth -- databricks current-user me diff --git a/acceptance/bundle/exec/databricks-cli/test.toml b/acceptance/bundle/exec/databricks-cli/test.toml new file mode 100644 index 0000000000..373138e0cd --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/test.toml @@ -0,0 +1,2 @@ +RecordRequests = true +IncludeRequestHeaders = ["Authorization"] diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index bb6ecfc118..4faac4039b 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -14,6 +14,27 @@ import ( // TODO: test that -- works with flags as well. +// TODO: Can bundle auth be resolved twice? What about: +// databricks bundle exec -t foo -- databricks jobs list -t bar? +// OR +// databricks bundle exec -- databricks jobs list -t bar? +// OR +// databricks bundle exec -- databricks jobs list? +// OR +// databricks bundle exec -t foo -- databricks jobs list? +// +// For the first two, undefined behavior is fine. For the latter two we need to ensure +// that the target from exec is respected. +// +// Add tests for all four of these cases. +// --> Do I need similar tests for --profile as well? +// --> Also add test for what happens with a default target? + +// TODO: Add acceptance test that flags are indeed not parsed by the exec command and +// instead are parsed by the child command. + +// # TODO: Table test casing the target permutations + func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", @@ -41,7 +62,13 @@ Examples: childCmd := exec.Command(args[0], args[1:]...) - childCmd.Env = auth.ProcessEnv(root.ConfigUsed(cmd.Context())) + env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) + // TODO: Test that this works correctly for all permutations. + // TODO: Do the same for profile flag. + // TODO: TODO: What happens here if a default target is resolved? When + // no targets are defined? + env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) + childCmd.Env = env // Execute all scripts from the bundle root directory. This behavior can // be surprising in isolation, but we do it to keep the behavior consistent From f0d1dc56c8c448ef25b49c09023b06a52a185f23 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 17:40:04 +0100 Subject: [PATCH 10/77] fix replacement --- .../bundle/exec/databricks-cli/basic/out.requests.txt | 2 +- acceptance/bundle/exec/databricks-cli/basic/script | 2 +- .../exec/databricks-cli/profile-is-passed/databricks.yml | 2 ++ .../bundle/exec/databricks-cli/profile-is-passed/output.txt | 3 +++ .../bundle/exec/databricks-cli/profile-is-passed/script | 0 acceptance/bundle/exec/databricks-cli/test.toml | 6 ++++++ cmd/bundle/exec.go | 2 +- 7 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/script diff --git a/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt b/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt index 10a364e97a..26efbbb08d 100644 --- a/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt +++ b/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt @@ -23,7 +23,7 @@ { "headers": { "Authorization": [ - "Basic Y2xpZW50X2lkOmNsaWVudC1zZWNyZXQ=" + "Basic [ENCODED_AUTH]" ] }, "method": "POST", diff --git a/acceptance/bundle/exec/databricks-cli/basic/script b/acceptance/bundle/exec/databricks-cli/basic/script index 3a3c5efe98..1f44239df4 100644 --- a/acceptance/bundle/exec/databricks-cli/basic/script +++ b/acceptance/bundle/exec/databricks-cli/basic/script @@ -11,5 +11,5 @@ trace errcode $CLI bundle exec -t pat -- databricks current-user me -t oauth # Explicitly select oauth target export DATABRICKS_TOKEN="" -export DATABRICKS_CLIENT_SECRET="client-secret" +export DATABRICKS_CLIENT_SECRET="client_secret" trace $CLI bundle exec -t oauth -- databricks current-user me diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml b/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml new file mode 100644 index 0000000000..432311dab0 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt new file mode 100644 index 0000000000..a258c7073a --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt @@ -0,0 +1,3 @@ +script: line 64: syntax error near unexpected token `)' + +Exit code: 2 diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/script new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/exec/databricks-cli/test.toml b/acceptance/bundle/exec/databricks-cli/test.toml index 373138e0cd..24cf889ece 100644 --- a/acceptance/bundle/exec/databricks-cli/test.toml +++ b/acceptance/bundle/exec/databricks-cli/test.toml @@ -1,2 +1,8 @@ RecordRequests = true IncludeRequestHeaders = ["Authorization"] + +# "client_id:client_secret" in base64 is Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ=, expect to +# see this in Authorization header +[[Repls]] +Old = "Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ=" +New = "[ENCODED_AUTH]" diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 4faac4039b..5ec6a0d280 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -33,7 +33,7 @@ import ( // TODO: Add acceptance test that flags are indeed not parsed by the exec command and // instead are parsed by the child command. -// # TODO: Table test casing the target permutations +// # TODO: Table test casing the profile flag permutations func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ From 080d1bf8b92ef3f80747284f04e0e6ca6daec5f8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 18:13:56 +0100 Subject: [PATCH 11/77] make the profile is passed test work --- .../profile-is-passed/.databrickscfg | 8 +++++ .../profile-is-passed/databricks.yml | 3 ++ .../profile-is-passed/out.requests.txt | 32 +++++++++++++++++++ .../profile-is-passed/output.txt | 13 ++++++-- .../databricks-cli/profile-is-passed/script | 14 ++++++++ cmd/bundle/exec.go | 8 +++++ 6 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg b/acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg new file mode 100644 index 0000000000..fc39190cbe --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg @@ -0,0 +1,8 @@ +[someprofile] +host = $DATABRICKS_HOST +token = $DATABRICKS_TOKEN + +[myprofile] +host = $DATABRICKS_HOST +client_id = client_id +client_secret = client_secret diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml b/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml index 432311dab0..5de7d1d968 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml @@ -1,2 +1,5 @@ bundle: name: foobar + +workspace: + profile: someprofile diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt new file mode 100644 index 0000000000..2b214d4fd2 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt @@ -0,0 +1,32 @@ +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "method": "GET", + "path": "/oidc/.well-known/oauth-authorization-server" +} +{ + "headers": { + "Authorization": [ + "Basic [ENCODED_AUTH]" + ] + }, + "method": "POST", + "path": "/oidc/v1/token", + "raw_body": "grant_type=client_credentials\u0026scope=all-apis" +} +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt index a258c7073a..d733dd0243 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt @@ -1,3 +1,12 @@ -script: line 64: syntax error near unexpected token `)' -Exit code: 2 +>>> [CLI] bundle exec -- databricks current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} + +>>> [CLI] bundle exec --profile myprofile -- databricks current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/script index e69de29bb2..ad9280c5a6 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/script @@ -0,0 +1,14 @@ +# Replace placeholder with an actual host URL +envsubst < databricks.yml > out.yml && mv out.yml databricks.yml +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Credentials will be picked up from .databrickscfg. Unset existing credentials. +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN + +# This should use the profile configured in the bundle, i.e. PAT auth +trace $CLI bundle exec -- databricks current-user me + +# This should use myprofile, which uses oauth. +trace $CLI bundle exec --profile myprofile -- databricks current-user me diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 5ec6a0d280..b5a52f6090 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -68,6 +68,14 @@ Examples: // TODO: TODO: What happens here if a default target is resolved? When // no targets are defined? env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) + + // If the bundle has a profile, explicitly pass it to the child command. + // This is unnecessary for tools that follow the unified authentication spec. + // However, because the CLI can read the profile from the bundle itself, we + // need to pass it explicitly. + if b.Config.Workspace.Profile != "" { + env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) + } childCmd.Env = env // Execute all scripts from the bundle root directory. This behavior can From 58d28a9c92c652ada851f4d5680f4d4ab8807bcc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 18:26:17 +0100 Subject: [PATCH 12/77] add more tests for profile and target --- .../{basic => target-is-passed}/databricks.yml | 0 .../{basic => target-is-passed}/out.requests.txt | 0 .../{basic => target-is-passed}/output.txt | 0 .../{basic => target-is-passed}/script | 0 bundle/config/mutator/default_target.go | 4 +++- cmd/bundle/exec.go | 16 +++++++++++----- 6 files changed, 14 insertions(+), 6 deletions(-) rename acceptance/bundle/exec/databricks-cli/{basic => target-is-passed}/databricks.yml (100%) rename acceptance/bundle/exec/databricks-cli/{basic => target-is-passed}/out.requests.txt (100%) rename acceptance/bundle/exec/databricks-cli/{basic => target-is-passed}/output.txt (100%) rename acceptance/bundle/exec/databricks-cli/{basic => target-is-passed}/script (100%) diff --git a/acceptance/bundle/exec/databricks-cli/basic/databricks.yml b/acceptance/bundle/exec/databricks-cli/target-is-passed/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/basic/databricks.yml rename to acceptance/bundle/exec/databricks-cli/target-is-passed/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/basic/out.requests.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/out.requests.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/basic/out.requests.txt rename to acceptance/bundle/exec/databricks-cli/target-is-passed/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/basic/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/basic/output.txt rename to acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt diff --git a/acceptance/bundle/exec/databricks-cli/basic/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/script similarity index 100% rename from acceptance/bundle/exec/databricks-cli/basic/script rename to acceptance/bundle/exec/databricks-cli/target-is-passed/script diff --git a/bundle/config/mutator/default_target.go b/bundle/config/mutator/default_target.go index 73d99002a0..781c075ecd 100644 --- a/bundle/config/mutator/default_target.go +++ b/bundle/config/mutator/default_target.go @@ -13,11 +13,13 @@ type defineDefaultTarget struct { name string } +const DefaultTargetName = "default" + // DefineDefaultTarget adds a target named "default" // to the configuration if none have been defined. func DefineDefaultTarget() bundle.Mutator { return &defineDefaultTarget{ - name: "default", + name: DefaultTargetName, } } diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index b5a52f6090..acf914f598 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/auth" "github.com/spf13/cobra" @@ -63,19 +64,24 @@ Examples: childCmd := exec.Command(args[0], args[1:]...) env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) - // TODO: Test that this works correctly for all permutations. - // TODO: Do the same for profile flag. - // TODO: TODO: What happens here if a default target is resolved? When - // no targets are defined? - env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) + + // If user has specified a target, pass it to the child command. If the + // target is the default target, we don't need to pass it explicitly since + // the CLI will use the default target by default. + // This is only useful for when the Databricks CLI is the child command. + if b.Config.Bundle.Target != mutator.DefaultTargetName { + env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) + } // If the bundle has a profile, explicitly pass it to the child command. // This is unnecessary for tools that follow the unified authentication spec. // However, because the CLI can read the profile from the bundle itself, we // need to pass it explicitly. + // This is only useful for when the Databricks CLI is the child command. if b.Config.Workspace.Profile != "" { env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } + childCmd.Env = env // Execute all scripts from the bundle root directory. This behavior can From 993956294aa35e707556bf486706f33ca9a85f51 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 18:35:03 +0100 Subject: [PATCH 13/77] added flags are not parsed check --- acceptance/bundle/exec/basic/output.txt | 3 +++ acceptance/bundle/exec/basic/script | 3 +++ cmd/bundle/exec.go | 26 ------------------------- 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index c1be99c9e6..01d3479f01 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -4,3 +4,6 @@ hello, world >>> [CLI] bundle exec -- pwd [TMPDIR] + +>>> [CLI] bundle exec -- echo --help +--help diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 48d20c1bd9..fb07fbfb6f 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -1,3 +1,6 @@ trace $CLI bundle exec -- echo "hello, world" trace $CLI bundle exec -- pwd + +# The CLI should not parse the --help flag and should pass it as is to echo. +trace $CLI bundle exec -- echo --help diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index acf914f598..39224955af 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -13,29 +13,6 @@ import ( "github.com/spf13/cobra" ) -// TODO: test that -- works with flags as well. - -// TODO: Can bundle auth be resolved twice? What about: -// databricks bundle exec -t foo -- databricks jobs list -t bar? -// OR -// databricks bundle exec -- databricks jobs list -t bar? -// OR -// databricks bundle exec -- databricks jobs list? -// OR -// databricks bundle exec -t foo -- databricks jobs list? -// -// For the first two, undefined behavior is fine. For the latter two we need to ensure -// that the target from exec is respected. -// -// Add tests for all four of these cases. -// --> Do I need similar tests for --profile as well? -// --> Also add test for what happens with a default target? - -// TODO: Add acceptance test that flags are indeed not parsed by the exec command and -// instead are parsed by the child command. - -// # TODO: Table test casing the profile flag permutations - func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", @@ -150,8 +127,5 @@ Examples: }, } - // TODO: Is this needed to make -- work with flags? What does this option do? - // execCmd.Flags().SetInterspersed(false) - return execCmd } From 05cd18c0be84c92c3f34de681fa4f17a22a835f6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 18:52:02 +0100 Subject: [PATCH 14/77] exit code done --- acceptance/bundle/exec/basic/output.txt | 5 +++++ acceptance/bundle/exec/basic/script | 3 +++ .../target-is-passed/output.txt | 2 +- cmd/bundle/exec.go | 20 ++++++++++++++++--- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 01d3479f01..3584582dba 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -7,3 +7,8 @@ hello, world >>> [CLI] bundle exec -- echo --help --help + +>>> [CLI] bundle exec -- bash -c exit 5 +Error: Running "bash -c exit 5" failed with exit code: 5 + +Exit code: 1 diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index fb07fbfb6f..41c57725f4 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -4,3 +4,6 @@ trace $CLI bundle exec -- pwd # The CLI should not parse the --help flag and should pass it as is to echo. trace $CLI bundle exec -- echo --help + +# The error message should include the exit code. +trace $CLI bundle exec -- bash -c "exit 5" diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt index 48fcf8a619..a17f177be6 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt @@ -13,7 +13,7 @@ >>> errcode [CLI] bundle exec -t pat -- databricks current-user me -t oauth Error: cannot resolve bundle auth configuration: validate: more than one authorization method configured: oauth and pat. Config: host=[DATABRICKS_URL], token=***, client_id=client_id, databricks_cli_path=[CLI]. Env: DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_CLI_PATH -Error: Command exited with code: 1 +Error: Running "databricks current-user me -t oauth" failed with exit code: 1 Exit code: 1 diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 39224955af..bd96b41dee 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -13,6 +13,15 @@ import ( "github.com/spf13/cobra" ) +type exitCodeErr struct { + exitCode int + args []string +} + +func (e *exitCodeErr) Error() string { + return fmt.Sprintf("Running %q failed with exit code: %d", strings.Join(e.args, " "), e.exitCode) +} + func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", @@ -110,11 +119,16 @@ Examples: }() // Wait for the command to finish. - // TODO: Pretty exit codes? - // TODO: Make CLI return the same exit codes? It has to, that's a requirement. err = childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { - return fmt.Errorf("Command exited with code: %d", exitErr.ExitCode()) + // We don't propagate the exit code as is because exit codes for + // the CLI have not been standardized yet. At some point in the + // future we might want to associate specific exit codes with + // specific classes of errors. + return &exitCodeErr{ + exitCode: exitErr.ExitCode(), + args: args, + } } if err != nil { return fmt.Errorf("Error waiting for command: %w", err) From a2748d531b31de50f7e46d729937d380c3c5c4af Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:19:39 +0100 Subject: [PATCH 15/77] cleanup --- acceptance/bundle/exec/basic/output.txt | 3 ++ acceptance/bundle/exec/basic/script | 5 +++- cmd/bundle/exec.go | 37 +++++++++++++++++++------ 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 3584582dba..9283131d6f 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -12,3 +12,6 @@ hello, world Error: Running "bash -c exit 5" failed with exit code: 5 Exit code: 1 + +>>> [CLI] bundle exec -- bash -c echo hello > /dev/stderr +hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 41c57725f4..ebe76884bd 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -6,4 +6,7 @@ trace $CLI bundle exec -- pwd trace $CLI bundle exec -- echo --help # The error message should include the exit code. -trace $CLI bundle exec -- bash -c "exit 5" +errcode trace $CLI bundle exec -- bash -c "exit 5" + +# stderr should also be shown in the output. +trace $CLI bundle exec -- bash -c "echo hello > /dev/stderr" diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index bd96b41dee..fbb4d25476 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -27,11 +27,12 @@ func newExecCommand() *cobra.Command { Use: "exec", Short: "Execute a command using the same authentication context as the bundle", Args: cobra.MinimumNArgs(1), - // TODO: format once we have all the documentation here. - Long: ` -Note: This command executes scripts + Long: `Execute a command using the same authentication context as the bundle -Examples: +The current working directory of the provided command will be set to the root +of the bundle. + +Example usage: 1. databricks bundle exec -- echo hello 2. databricks bundle exec -- /bin/bash -c "echo hello"" 3. databricks bundle exec -- uv run pytest"`, @@ -81,8 +82,8 @@ Examples: // adding support for the scripts section. childCmd.Dir = b.BundleRootPath - // Create pipes for stdout and stderr. - // TODO: Test streaming of this? Is there a way? + // Create pipes to stream the stdout and stderr output from the child + // process. stdout, err := childCmd.StdoutPipe() if err != nil { return fmt.Errorf("Error creating stdout pipe: %w", err) @@ -98,15 +99,21 @@ Examples: return fmt.Errorf("Error starting command: %s\n", err) } - // Stream both stdout and stderr to the user. + // Stream both stdout and stderr to the user. We do this so that the user + // does not have to wait for the command to finish before seeing the output. var wg sync.WaitGroup + var stdoutErr, stderrErr error wg.Add(2) go func() { defer wg.Done() scanner := bufio.NewScanner(stdout) for scanner.Scan() { - fmt.Println(scanner.Text()) + _, err = cmd.OutOrStdout().Write([]byte(scanner.Text() + "\n")) + if err != nil { + stdoutErr = fmt.Errorf("Error writing to stdout: %w", err) + return + } } }() @@ -114,10 +121,22 @@ Examples: defer wg.Done() scanner := bufio.NewScanner(stderr) for scanner.Scan() { - fmt.Println(scanner.Text()) + _, err := cmd.ErrOrStderr().Write([]byte(scanner.Text() + "\n")) + if err != nil { + stderrErr = fmt.Errorf("Error writing to stderr: %w", err) + return + } } }() + if stdoutErr != nil { + return stdoutErr + } + + if stderrErr != nil { + return stderrErr + } + // Wait for the command to finish. err = childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { From 1996d3f824ef52c6a9f0715fd171e4285ee4ab81 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:29:19 +0100 Subject: [PATCH 16/77] do not run on cloud --- acceptance/bundle/exec/no-auth/script | 4 ++-- acceptance/bundle/exec/test.toml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/exec/test.toml diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/exec/no-auth/script index 93715d4481..0378ab2663 100644 --- a/acceptance/bundle/exec/no-auth/script +++ b/acceptance/bundle/exec/no-auth/script @@ -1,5 +1,5 @@ -export DATABRICKS_HOST="" -export DATABRICKS_TOKEN="" +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN # Confirm that bundle exec works for commands that do not require authentication, # even if authentication is not provided. diff --git a/acceptance/bundle/exec/test.toml b/acceptance/bundle/exec/test.toml new file mode 100644 index 0000000000..4564a92fed --- /dev/null +++ b/acceptance/bundle/exec/test.toml @@ -0,0 +1,2 @@ +Cloud = false +Local = true From 88b6dc8e16a366dc2da91e3368f88ef9ca3d7da1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:31:41 +0100 Subject: [PATCH 17/77] use $CLI instead of databricks --- .../exec/databricks-cli/profile-is-passed/output.txt | 4 ++-- .../exec/databricks-cli/profile-is-passed/script | 4 ++-- .../exec/databricks-cli/target-is-passed/output.txt | 10 +++++----- .../bundle/exec/databricks-cli/target-is-passed/script | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt index d733dd0243..06cdd1c440 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt @@ -1,11 +1,11 @@ ->>> [CLI] bundle exec -- databricks current-user me +>>> [CLI] bundle exec -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" } ->>> [CLI] bundle exec --profile myprofile -- databricks current-user me +>>> [CLI] bundle exec --profile myprofile -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/script index ad9280c5a6..33e07494c6 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/script @@ -8,7 +8,7 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # This should use the profile configured in the bundle, i.e. PAT auth -trace $CLI bundle exec -- databricks current-user me +trace $CLI bundle exec -- $CLI current-user me # This should use myprofile, which uses oauth. -trace $CLI bundle exec --profile myprofile -- databricks current-user me +trace $CLI bundle exec --profile myprofile -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt index a17f177be6..84a9653e92 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt @@ -1,23 +1,23 @@ ->>> [CLI] bundle exec -- databricks current-user me +>>> [CLI] bundle exec -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" } ->>> [CLI] bundle exec -t pat -- databricks current-user me +>>> [CLI] bundle exec -t pat -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" } ->>> errcode [CLI] bundle exec -t pat -- databricks current-user me -t oauth +>>> errcode [CLI] bundle exec -t pat -- [CLI] current-user me -t oauth Error: cannot resolve bundle auth configuration: validate: more than one authorization method configured: oauth and pat. Config: host=[DATABRICKS_URL], token=***, client_id=client_id, databricks_cli_path=[CLI]. Env: DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_CLI_PATH -Error: Running "databricks current-user me -t oauth" failed with exit code: 1 +Error: Running "[CLI] current-user me -t oauth" failed with exit code: 1 Exit code: 1 ->>> [CLI] bundle exec -t oauth -- databricks current-user me +>>> [CLI] bundle exec -t oauth -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/script index 1f44239df4..9d2eb2fe7a 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/script @@ -1,15 +1,15 @@ # Default target -trace $CLI bundle exec -- databricks current-user me +trace $CLI bundle exec -- $CLI current-user me # Explicitly select default target -trace $CLI bundle exec -t pat -- databricks current-user me +trace $CLI bundle exec -t pat -- $CLI current-user me # Conflicting targets selected. This should fail because for the child command # pat would be configured via environment variables and oauth via the CLI resulting # in more than one authorization method configured. -trace errcode $CLI bundle exec -t pat -- databricks current-user me -t oauth +trace errcode $CLI bundle exec -t pat -- $CLI current-user me -t oauth # Explicitly select oauth target export DATABRICKS_TOKEN="" export DATABRICKS_CLIENT_SECRET="client_secret" -trace $CLI bundle exec -t oauth -- databricks current-user me +trace $CLI bundle exec -t oauth -- $CLI current-user me From fd2600cbea9438dded4503062c95026ccbb34cea Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:32:43 +0100 Subject: [PATCH 18/77] return stdout / stderr errors after --- cmd/bundle/exec.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index fbb4d25476..f74458d06a 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -129,14 +129,6 @@ Example usage: } }() - if stdoutErr != nil { - return stdoutErr - } - - if stderrErr != nil { - return stderrErr - } - // Wait for the command to finish. err = childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { @@ -156,6 +148,14 @@ Example usage: // Wait for the goroutines to finish printing to stdout and stderr. wg.Wait() + if stdoutErr != nil { + return stdoutErr + } + + if stderrErr != nil { + return stderrErr + } + return nil }, } From 93c5e3f2ae648afd701cb958d80106ce9ea7ef0f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:50:56 +0100 Subject: [PATCH 19/77] simplify streaming --- cmd/bundle/exec.go | 59 ++++------------------------------------------ 1 file changed, 4 insertions(+), 55 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index f74458d06a..794d2aa0ec 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -1,11 +1,9 @@ package bundle import ( - "bufio" "fmt" "os/exec" "strings" - "sync" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/cmd/root" @@ -82,55 +80,17 @@ Example usage: // adding support for the scripts section. childCmd.Dir = b.BundleRootPath - // Create pipes to stream the stdout and stderr output from the child - // process. - stdout, err := childCmd.StdoutPipe() - if err != nil { - return fmt.Errorf("Error creating stdout pipe: %w", err) - } - - stderr, err := childCmd.StderrPipe() - if err != nil { - return fmt.Errorf("Error creating stderr pipe: %w", err) - } + // Stream the stdout and stderr of the child process directly. + childCmd.Stdout = cmd.OutOrStdout() + childCmd.Stderr = cmd.ErrOrStderr() // Start the command if err := childCmd.Start(); err != nil { return fmt.Errorf("Error starting command: %s\n", err) } - // Stream both stdout and stderr to the user. We do this so that the user - // does not have to wait for the command to finish before seeing the output. - var wg sync.WaitGroup - var stdoutErr, stderrErr error - wg.Add(2) - - go func() { - defer wg.Done() - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - _, err = cmd.OutOrStdout().Write([]byte(scanner.Text() + "\n")) - if err != nil { - stdoutErr = fmt.Errorf("Error writing to stdout: %w", err) - return - } - } - }() - - go func() { - defer wg.Done() - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - _, err := cmd.ErrOrStderr().Write([]byte(scanner.Text() + "\n")) - if err != nil { - stderrErr = fmt.Errorf("Error writing to stderr: %w", err) - return - } - } - }() - // Wait for the command to finish. - err = childCmd.Wait() + err := childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { // We don't propagate the exit code as is because exit codes for // the CLI have not been standardized yet. At some point in the @@ -145,17 +105,6 @@ Example usage: return fmt.Errorf("Error waiting for command: %w", err) } - // Wait for the goroutines to finish printing to stdout and stderr. - wg.Wait() - - if stdoutErr != nil { - return stdoutErr - } - - if stderrErr != nil { - return stderrErr - } - return nil }, } From bbbbf30db3b41dceddc847af7b75e99161d1e8bc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:51:39 +0100 Subject: [PATCH 20/77] - --- .vscode/settings.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index f103538b7f..f8b04f1269 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,8 +17,5 @@ "python.envFile": "${workspaceRoot}/.env", "python.analysis.stubPath": ".vscode", "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", - "files.associations": { - "script": "shellscript" - } + "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------" } From eb58d11f9b247a248e11f3da59595615eeb4fa15 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:52:45 +0100 Subject: [PATCH 21/77] - --- acceptance/bundle/exec/basic/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index ebe76884bd..e0e238c43d 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -8,5 +8,5 @@ trace $CLI bundle exec -- echo --help # The error message should include the exit code. errcode trace $CLI bundle exec -- bash -c "exit 5" -# stderr should also be shown in the output. +# stderr should also be passed through. trace $CLI bundle exec -- bash -c "echo hello > /dev/stderr" From 007a714750ee2347728a85ff86d45c49b6fdf835 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:56:29 +0100 Subject: [PATCH 22/77] - --- acceptance/bundle/exec/databricks-cli/target-is-passed/script | 2 +- acceptance/bundle/exec/no-auth/script | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/script index 9d2eb2fe7a..238c36ae4b 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/script @@ -10,6 +10,6 @@ trace $CLI bundle exec -t pat -- $CLI current-user me trace errcode $CLI bundle exec -t pat -- $CLI current-user me -t oauth # Explicitly select oauth target -export DATABRICKS_TOKEN="" +unset DATABRICKS_TOKEN export DATABRICKS_CLIENT_SECRET="client_secret" trace $CLI bundle exec -t oauth -- $CLI current-user me diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/exec/no-auth/script index 0378ab2663..a1758ce997 100644 --- a/acceptance/bundle/exec/no-auth/script +++ b/acceptance/bundle/exec/no-auth/script @@ -4,5 +4,4 @@ unset DATABRICKS_TOKEN # Confirm that bundle exec works for commands that do not require authentication, # even if authentication is not provided. trace $CLI bundle exec -- echo hello - trace $CLI bundle exec -- pwd From 0c9fbf7b236510d208183b8c358b9e33fa7a7797 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 19:58:14 +0100 Subject: [PATCH 23/77] - --- bundle/config/mutator/default_target.go | 4 ++-- cmd/bundle/exec.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/default_target.go b/bundle/config/mutator/default_target.go index 781c075ecd..c2e9e8acf2 100644 --- a/bundle/config/mutator/default_target.go +++ b/bundle/config/mutator/default_target.go @@ -13,13 +13,13 @@ type defineDefaultTarget struct { name string } -const DefaultTargetName = "default" +const DefaultTargetPlaceholder = "default" // DefineDefaultTarget adds a target named "default" // to the configuration if none have been defined. func DefineDefaultTarget() bundle.Mutator { return &defineDefaultTarget{ - name: DefaultTargetName, + name: DefaultTargetPlaceholder, } } diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 794d2aa0ec..4c62ec292f 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -54,7 +54,7 @@ Example usage: // target is the default target, we don't need to pass it explicitly since // the CLI will use the default target by default. // This is only useful for when the Databricks CLI is the child command. - if b.Config.Bundle.Target != mutator.DefaultTargetName { + if b.Config.Bundle.Target != mutator.DefaultTargetPlaceholder { env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) } From edf705188d936f48113de822f2662b7b360f4c23 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:06:24 +0100 Subject: [PATCH 24/77] more cleanup --- cmd/bundle/exec.go | 33 ++++++++++++++++++--------------- internal/testutil/env.go | 31 +++++++++++-------------------- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 4c62ec292f..1463aa2027 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -31,9 +31,9 @@ The current working directory of the provided command will be set to the root of the bundle. Example usage: -1. databricks bundle exec -- echo hello -2. databricks bundle exec -- /bin/bash -c "echo hello"" -3. databricks bundle exec -- uv run pytest"`, +1. databricks bundle exec -- echo "hello, world" +2. databricks bundle exec -- /bin/bash -c "echo hello" +3. databricks bundle exec -- uv run pytest`, RunE: func(cmd *cobra.Command, args []string) error { if cmd.ArgsLenAtDash() != 0 { return fmt.Errorf("Please add a '--' separator. Usage: 'databricks bundle exec -- %s'", strings.Join(args, " ")) @@ -50,19 +50,20 @@ Example usage: env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) - // If user has specified a target, pass it to the child command. If the - // target is the default target, we don't need to pass it explicitly since - // the CLI will use the default target by default. + // If user has specified a target, pass it to the child command. DABs + // defines a "default" target which is a placeholder if no target is defined. + // If that's the case, i.e. no targets are defined, then do not pass the target. + // // This is only useful for when the Databricks CLI is the child command. if b.Config.Bundle.Target != mutator.DefaultTargetPlaceholder { env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) } - // If the bundle has a profile, explicitly pass it to the child command. - // This is unnecessary for tools that follow the unified authentication spec. - // However, because the CLI can read the profile from the bundle itself, we - // need to pass it explicitly. - // This is only useful for when the Databricks CLI is the child command. + // If the bundle has a profile configured, explicitly pass it to the child command. + // + // This is only useful for when the Databricks CLI is the child command, + // since if we do not explicitly pass the profile, the CLI will use the + // profile configured in the bundle YAML configuration (if any).We don't propagate the exit code as is because exit codes if b.Config.Workspace.Profile != "" { env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } @@ -92,10 +93,12 @@ Example usage: // Wait for the command to finish. err := childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { - // We don't propagate the exit code as is because exit codes for - // the CLI have not been standardized yet. At some point in the - // future we might want to associate specific exit codes with - // specific classes of errors. + // We don't make the parent CLI process exit with the same exit code + // as the child process because the exit codes for the CLI have not + // been standardized yet. + // + // This keeps the door open for us to associate specific exit codes + // with specific classes of errors in the future. return &exitCodeErr{ exitCode: exitErr.ExitCode(), args: args, diff --git a/internal/testutil/env.go b/internal/testutil/env.go index 0aa97d7799..598229655f 100644 --- a/internal/testutil/env.go +++ b/internal/testutil/env.go @@ -13,11 +13,19 @@ import ( // The original environment is restored upon test completion. // Note: use of this function is incompatible with parallel execution. func CleanupEnvironment(t TestingT) { + // Restore environment when test finishes. + environ := os.Environ() + t.Cleanup(func() { + // Restore original environment. + for _, kv := range environ { + kvs := strings.SplitN(kv, "=", 2) + os.Setenv(kvs[0], kvs[1]) + } + }) + path := os.Getenv("PATH") pwd := os.Getenv("PWD") - - // Clear all environment variables. - ClearEnvironment(t) + os.Clearenv() // We use t.Setenv instead of os.Setenv because the former actively // prevents a test being run with t.Parallel. Modifying the environment @@ -30,23 +38,6 @@ func CleanupEnvironment(t TestingT) { } } -// ClearEnvironment sets up an empty environment with no environment variables set. -// The original environment is restored upon test completion. -// Note: use of this function is incompatible with parallel execution -func ClearEnvironment(t TestingT) { - // Restore environment when test finishes. - environ := os.Environ() - t.Cleanup(func() { - // Restore original environment. - for _, kv := range environ { - kvs := strings.SplitN(kv, "=", 2) - os.Setenv(kvs[0], kvs[1]) - } - }) - - os.Clearenv() -} - // Changes into specified directory for the duration of the test. // Returns the current working directory. func Chdir(t TestingT, dir string) string { From 769acf7fd3ffb1730850a45c0f0db276d2f8a4af Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:07:51 +0100 Subject: [PATCH 25/77] - --- cmd/bundle/exec.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 1463aa2027..8703b4fbc7 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -39,8 +39,7 @@ Example usage: return fmt.Errorf("Please add a '--' separator. Usage: 'databricks bundle exec -- %s'", strings.Join(args, " ")) } - // Load the bundle configuration to get the authentication credentials - // set in the context. + // Load the bundle configuration to get the authentication credentials. b, diags := root.MustConfigureBundle(cmd) if diags.HasError() { return diags.Error() From 406103623d56c867c1aa25d13277d96cb05f482f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:08:33 +0100 Subject: [PATCH 26/77] - --- cmd/bundle/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 8703b4fbc7..0010d62312 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -50,7 +50,7 @@ Example usage: env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) // If user has specified a target, pass it to the child command. DABs - // defines a "default" target which is a placeholder if no target is defined. + // defines a "default" target which is a placeholder for when no target is defined. // If that's the case, i.e. no targets are defined, then do not pass the target. // // This is only useful for when the Databricks CLI is the child command. From c0d34f7827d42faf4b488793c0d28e76849233e2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:09:20 +0100 Subject: [PATCH 27/77] - --- cmd/bundle/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 0010d62312..3fd2b37d31 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -62,7 +62,7 @@ Example usage: // // This is only useful for when the Databricks CLI is the child command, // since if we do not explicitly pass the profile, the CLI will use the - // profile configured in the bundle YAML configuration (if any).We don't propagate the exit code as is because exit codes + // profile configured in the bundle YAML configuration (if any). if b.Config.Workspace.Profile != "" { env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } From 112a9de90a5dbe97bea17fc09e20e0a46141b403 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:10:27 +0100 Subject: [PATCH 28/77] - --- cmd/bundle/exec.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 3fd2b37d31..7dd831a200 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -71,9 +71,9 @@ Example usage: // Execute all scripts from the bundle root directory. This behavior can // be surprising in isolation, but we do it to keep the behavior consistent - // for both cases: + // for both these cases: // 1. One shot commands like `databricks bundle exec -- echo hello` - // 2. Scripts that are defined in the scripts section of the DAB. + // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. // // TODO(shreyas): Add a DATABRICKS_BUNDLE_INITIAL_CWD environment variable // that users can read to figure out the original CWD. I'll do that when From 647dab0a6660d619824836b58d0eeb857669abcd Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Mar 2025 20:16:12 +0100 Subject: [PATCH 29/77] some cleanup --- .../bundle/exec/cmd-not-found/databricks.yml | 2 ++ .../bundle/exec/cmd-not-found/output.txt | 5 ++++ acceptance/bundle/exec/cmd-not-found/script | 1 + cmd/bundle/exec.go | 23 ++++++++++++------- 4 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 acceptance/bundle/exec/cmd-not-found/databricks.yml create mode 100644 acceptance/bundle/exec/cmd-not-found/output.txt create mode 100644 acceptance/bundle/exec/cmd-not-found/script diff --git a/acceptance/bundle/exec/cmd-not-found/databricks.yml b/acceptance/bundle/exec/cmd-not-found/databricks.yml new file mode 100644 index 0000000000..432311dab0 --- /dev/null +++ b/acceptance/bundle/exec/cmd-not-found/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/exec/cmd-not-found/output.txt new file mode 100644 index 0000000000..36db646676 --- /dev/null +++ b/acceptance/bundle/exec/cmd-not-found/output.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 +Error: Running "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in $PATH + +Exit code: 1 diff --git a/acceptance/bundle/exec/cmd-not-found/script b/acceptance/bundle/exec/cmd-not-found/script new file mode 100644 index 0000000000..4a5d442f3a --- /dev/null +++ b/acceptance/bundle/exec/cmd-not-found/script @@ -0,0 +1 @@ +trace $CLI bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 7dd831a200..dbd7d11569 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -20,6 +20,15 @@ func (e *exitCodeErr) Error() string { return fmt.Sprintf("Running %q failed with exit code: %d", strings.Join(e.args, " "), e.exitCode) } +type runErr struct { + err error + args []string +} + +func (e *runErr) Error() string { + return fmt.Sprintf("Running %q failed: %s", strings.Join(e.args, " "), e.err) +} + func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", @@ -84,13 +93,8 @@ Example usage: childCmd.Stdout = cmd.OutOrStdout() childCmd.Stderr = cmd.ErrOrStderr() - // Start the command - if err := childCmd.Start(); err != nil { - return fmt.Errorf("Error starting command: %s\n", err) - } - - // Wait for the command to finish. - err := childCmd.Wait() + // Run the command. + err := childCmd.Run() if exitErr, ok := err.(*exec.ExitError); ok { // We don't make the parent CLI process exit with the same exit code // as the child process because the exit codes for the CLI have not @@ -104,7 +108,10 @@ Example usage: } } if err != nil { - return fmt.Errorf("Error waiting for command: %w", err) + return &runErr{ + err: err, + args: args, + } } return nil From be3be9cb2f1a212c5c881c6018dd6fbd0416b8d4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Mar 2025 18:22:12 +0100 Subject: [PATCH 30/77] try streaming output --- .../bundle/exec/cmd-not-found/output.txt | 2 +- .../profile-is-passed/output.txt | 8 +- .../target-is-passed/output.txt | 12 +-- cmd/bundle/exec.go | 76 ++++++++++++++++--- 4 files changed, 78 insertions(+), 20 deletions(-) diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/exec/cmd-not-found/output.txt index 36db646676..8344948b44 100644 --- a/acceptance/bundle/exec/cmd-not-found/output.txt +++ b/acceptance/bundle/exec/cmd-not-found/output.txt @@ -1,5 +1,5 @@ >>> [CLI] bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 -Error: Running "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in $PATH +Error: starting "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in $PATH Exit code: 1 diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt index 06cdd1c440..1c1b740582 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt @@ -1,12 +1,12 @@ >>> [CLI] bundle exec -- [CLI] current-user me { - "id":"[USERID]", - "userName":"[USERNAME]" +"id":"[USERID]", +"userName":"[USERNAME]" } >>> [CLI] bundle exec --profile myprofile -- [CLI] current-user me { - "id":"[USERID]", - "userName":"[USERNAME]" +"id":"[USERID]", +"userName":"[USERNAME]" } diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt index 84a9653e92..0fce7c1a72 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt @@ -1,14 +1,14 @@ >>> [CLI] bundle exec -- [CLI] current-user me { - "id":"[USERID]", - "userName":"[USERNAME]" +"id":"[USERID]", +"userName":"[USERNAME]" } >>> [CLI] bundle exec -t pat -- [CLI] current-user me { - "id":"[USERID]", - "userName":"[USERNAME]" +"id":"[USERID]", +"userName":"[USERNAME]" } >>> errcode [CLI] bundle exec -t pat -- [CLI] current-user me -t oauth @@ -19,6 +19,6 @@ Exit code: 1 >>> [CLI] bundle exec -t oauth -- [CLI] current-user me { - "id":"[USERID]", - "userName":"[USERNAME]" +"id":"[USERID]", +"userName":"[USERNAME]" } diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index dbd7d11569..e914f88448 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -1,9 +1,11 @@ package bundle import ( + "bufio" "fmt" "os/exec" "strings" + "sync" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/cmd/root" @@ -89,12 +91,71 @@ Example usage: // adding support for the scripts section. childCmd.Dir = b.BundleRootPath - // Stream the stdout and stderr of the child process directly. - childCmd.Stdout = cmd.OutOrStdout() - childCmd.Stderr = cmd.ErrOrStderr() + stdout, err := childCmd.StdoutPipe() + if err != nil { + return fmt.Errorf("creating stdout pipe failed: %w", err) + } + + stderr, err := childCmd.StderrPipe() + if err != nil { + return fmt.Errorf("creating stderr pipe failed: %w", err) + } + + // Start the child command. + err = childCmd.Start() + if err != nil { + return fmt.Errorf("starting %q failed: %w", strings.Join(args, " "), err) + } + + var wg sync.WaitGroup + wg.Add(2) + + var stdoutErr error + go func() { + reader := bufio.NewReader(stdout) + line, err := reader.ReadString('\n') + for err == nil { + _, err = fmt.Fprintf(cmd.OutOrStdout(), "%s\n", strings.TrimSpace(line)) + if err != nil { + stdoutErr = err + break + } + line, err = reader.ReadString('\n') + } - // Run the command. - err := childCmd.Run() + wg.Done() + }() + + var stderrErr error + go func() { + reader := bufio.NewReader(stderr) + // TODO CONTINUE: The formatting is messed u[] because of the new line business + // here. + // Fix that. + line, err := reader.ReadString('\n') + for err == nil { + _, err = fmt.Fprintf(cmd.ErrOrStderr(), "%s\n", strings.TrimSpace(line)) + if err != nil { + stderrErr = err + break + } + line, err = reader.ReadString('\n') + } + + wg.Done() + }() + + wg.Wait() + + if stdoutErr != nil { + return fmt.Errorf("writing stdout failed: %w", stdoutErr) + } + + if stderrErr != nil { + return fmt.Errorf("writing stderr failed: %w", stderrErr) + } + + err = childCmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { // We don't make the parent CLI process exit with the same exit code // as the child process because the exit codes for the CLI have not @@ -108,10 +169,7 @@ Example usage: } } if err != nil { - return &runErr{ - err: err, - args: args, - } + return fmt.Errorf("running %q failed: %w", strings.Join(args, " "), err) } return nil From 39d0b13c4f1a25f61a787bced2157f2717d98deb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 10:29:57 +0100 Subject: [PATCH 31/77] - --- libs/auth/env.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libs/auth/env.go b/libs/auth/env.go index ddf73c9868..08282e4634 100644 --- a/libs/auth/env.go +++ b/libs/auth/env.go @@ -4,10 +4,6 @@ import ( "fmt" "os" "slices" -<<<<<<< HEAD - "sort" -======= ->>>>>>> origin "strings" "github.com/databricks/databricks-sdk-go/config" From 2212aa25973e6af1f62247d16ab4c08f3eac6f13 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 10:44:43 +0100 Subject: [PATCH 32/77] proper printing --- .../profile-is-passed/output.txt | 8 +++--- .../target-is-passed/output.txt | 12 ++++----- cmd/bundle/exec.go | 27 +++++++------------ 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt index 1c1b740582..06cdd1c440 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt @@ -1,12 +1,12 @@ >>> [CLI] bundle exec -- [CLI] current-user me { -"id":"[USERID]", -"userName":"[USERNAME]" + "id":"[USERID]", + "userName":"[USERNAME]" } >>> [CLI] bundle exec --profile myprofile -- [CLI] current-user me { -"id":"[USERID]", -"userName":"[USERNAME]" + "id":"[USERID]", + "userName":"[USERNAME]" } diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt index 0fce7c1a72..84a9653e92 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt @@ -1,14 +1,14 @@ >>> [CLI] bundle exec -- [CLI] current-user me { -"id":"[USERID]", -"userName":"[USERNAME]" + "id":"[USERID]", + "userName":"[USERNAME]" } >>> [CLI] bundle exec -t pat -- [CLI] current-user me { -"id":"[USERID]", -"userName":"[USERNAME]" + "id":"[USERID]", + "userName":"[USERNAME]" } >>> errcode [CLI] bundle exec -t pat -- [CLI] current-user me -t oauth @@ -19,6 +19,6 @@ Exit code: 1 >>> [CLI] bundle exec -t oauth -- [CLI] current-user me { -"id":"[USERID]", -"userName":"[USERNAME]" + "id":"[USERID]", + "userName":"[USERNAME]" } diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index e914f88448..4a1d7b0ebe 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -112,37 +112,30 @@ Example usage: var stdoutErr error go func() { - reader := bufio.NewReader(stdout) - line, err := reader.ReadString('\n') - for err == nil { - _, err = fmt.Fprintf(cmd.OutOrStdout(), "%s\n", strings.TrimSpace(line)) + defer wg.Done() + + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + _, err = fmt.Fprintln(cmd.OutOrStdout(), scanner.Text()) if err != nil { stdoutErr = err break } - line, err = reader.ReadString('\n') } - - wg.Done() }() var stderrErr error go func() { - reader := bufio.NewReader(stderr) - // TODO CONTINUE: The formatting is messed u[] because of the new line business - // here. - // Fix that. - line, err := reader.ReadString('\n') - for err == nil { - _, err = fmt.Fprintf(cmd.ErrOrStderr(), "%s\n", strings.TrimSpace(line)) + defer wg.Done() + + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + _, err = fmt.Fprintln(cmd.ErrOrStderr(), scanner.Text()) if err != nil { stderrErr = err break } - line, err = reader.ReadString('\n') } - - wg.Done() }() wg.Wait() From 7fb464b49627572d8fc1b28d8b54a3a4b6a5e624 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 10:47:58 +0100 Subject: [PATCH 33/77] lint --- cmd/bundle/exec.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 4a1d7b0ebe..2bf925820d 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -22,15 +22,6 @@ func (e *exitCodeErr) Error() string { return fmt.Sprintf("Running %q failed with exit code: %d", strings.Join(e.args, " "), e.exitCode) } -type runErr struct { - err error - args []string -} - -func (e *runErr) Error() string { - return fmt.Sprintf("Running %q failed: %s", strings.Join(e.args, " "), e.err) -} - func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", From b5e21231eb1029f83f36575419a7df0dc54f7186 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 10:49:47 +0100 Subject: [PATCH 34/77] - --- acceptance/bundle/exec/basic/output.txt | 10 ++-------- acceptance/bundle/exec/basic/script | 5 +++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 9283131d6f..c9a651aed0 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -5,13 +5,7 @@ hello, world >>> [CLI] bundle exec -- pwd [TMPDIR] ->>> [CLI] bundle exec -- echo --help ---help - ->>> [CLI] bundle exec -- bash -c exit 5 -Error: Running "bash -c exit 5" failed with exit code: 5 +>>> [CLI] bundle exec -- --help +Error: starting "--help" failed: exec: "--help": executable file not found in $PATH Exit code: 1 - ->>> [CLI] bundle exec -- bash -c echo hello > /dev/stderr -hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index e0e238c43d..9d1a281c2b 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -2,8 +2,9 @@ trace $CLI bundle exec -- echo "hello, world" trace $CLI bundle exec -- pwd -# The CLI should not parse the --help flag and should pass it as is to echo. -trace $CLI bundle exec -- echo --help +# The CLI should not parse the --help flag and should try to run it as an executable +# instead. +trace $CLI bundle exec -- --help # The error message should include the exit code. errcode trace $CLI bundle exec -- bash -c "exit 5" From 39ec48a602427ecf303d845ad0bef7e0a06ea63e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 10:58:18 +0100 Subject: [PATCH 35/77] remove error struct --- .../databricks-cli/target-is-passed/output.txt | 2 +- cmd/bundle/exec.go | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt index 84a9653e92..0751ac7dc0 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt @@ -13,7 +13,7 @@ >>> errcode [CLI] bundle exec -t pat -- [CLI] current-user me -t oauth Error: cannot resolve bundle auth configuration: validate: more than one authorization method configured: oauth and pat. Config: host=[DATABRICKS_URL], token=***, client_id=client_id, databricks_cli_path=[CLI]. Env: DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_CLI_PATH -Error: Running "[CLI] current-user me -t oauth" failed with exit code: 1 +Error: running "[CLI] current-user me -t oauth" failed with exit code: 1 Exit code: 1 diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 2bf925820d..bbb8038b95 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -13,15 +13,6 @@ import ( "github.com/spf13/cobra" ) -type exitCodeErr struct { - exitCode int - args []string -} - -func (e *exitCodeErr) Error() string { - return fmt.Sprintf("Running %q failed with exit code: %d", strings.Join(e.args, " "), e.exitCode) -} - func newExecCommand() *cobra.Command { execCmd := &cobra.Command{ Use: "exec", @@ -147,10 +138,7 @@ Example usage: // // This keeps the door open for us to associate specific exit codes // with specific classes of errors in the future. - return &exitCodeErr{ - exitCode: exitErr.ExitCode(), - args: args, - } + return fmt.Errorf("running %q failed with exit code: %d", strings.Join(args, " "), exitErr.ExitCode()) } if err != nil { return fmt.Errorf("running %q failed: %w", strings.Join(args, " "), err) From 2c773683bc3133d11fd0cf04c2ca5de0431db4a8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:08:09 +0100 Subject: [PATCH 36/77] split windows test --- acceptance/bundle/exec/basic/test.toml | 3 +++ acceptance/bundle/exec/cmd-not-found/test.toml | 3 +++ acceptance/bundle/exec/cwd/{ => unix}/a/b/c/.gitkeep | 0 acceptance/bundle/exec/cwd/{ => unix}/databricks.yml | 0 acceptance/bundle/exec/cwd/{ => unix}/output.txt | 0 acceptance/bundle/exec/cwd/{ => unix}/script | 0 acceptance/bundle/exec/cwd/unix/test.toml | 4 ++++ acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep | 0 acceptance/bundle/exec/cwd/windows/databricks.yml | 2 ++ acceptance/bundle/exec/cwd/windows/output.txt | 8 ++++++++ acceptance/bundle/exec/cwd/windows/script | 6 ++++++ acceptance/bundle/exec/cwd/windows/test.toml | 4 ++++ 12 files changed, 30 insertions(+) create mode 100644 acceptance/bundle/exec/basic/test.toml create mode 100644 acceptance/bundle/exec/cmd-not-found/test.toml rename acceptance/bundle/exec/cwd/{ => unix}/a/b/c/.gitkeep (100%) rename acceptance/bundle/exec/cwd/{ => unix}/databricks.yml (100%) rename acceptance/bundle/exec/cwd/{ => unix}/output.txt (100%) rename acceptance/bundle/exec/cwd/{ => unix}/script (100%) create mode 100644 acceptance/bundle/exec/cwd/unix/test.toml create mode 100644 acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep create mode 100644 acceptance/bundle/exec/cwd/windows/databricks.yml create mode 100644 acceptance/bundle/exec/cwd/windows/output.txt create mode 100644 acceptance/bundle/exec/cwd/windows/script create mode 100644 acceptance/bundle/exec/cwd/windows/test.toml diff --git a/acceptance/bundle/exec/basic/test.toml b/acceptance/bundle/exec/basic/test.toml new file mode 100644 index 0000000000..ca7d069ae6 --- /dev/null +++ b/acceptance/bundle/exec/basic/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = "%PATH%" +New = "$PATH" diff --git a/acceptance/bundle/exec/cmd-not-found/test.toml b/acceptance/bundle/exec/cmd-not-found/test.toml new file mode 100644 index 0000000000..ca7d069ae6 --- /dev/null +++ b/acceptance/bundle/exec/cmd-not-found/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = "%PATH%" +New = "$PATH" diff --git a/acceptance/bundle/exec/cwd/a/b/c/.gitkeep b/acceptance/bundle/exec/cwd/unix/a/b/c/.gitkeep similarity index 100% rename from acceptance/bundle/exec/cwd/a/b/c/.gitkeep rename to acceptance/bundle/exec/cwd/unix/a/b/c/.gitkeep diff --git a/acceptance/bundle/exec/cwd/databricks.yml b/acceptance/bundle/exec/cwd/unix/databricks.yml similarity index 100% rename from acceptance/bundle/exec/cwd/databricks.yml rename to acceptance/bundle/exec/cwd/unix/databricks.yml diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/exec/cwd/unix/output.txt similarity index 100% rename from acceptance/bundle/exec/cwd/output.txt rename to acceptance/bundle/exec/cwd/unix/output.txt diff --git a/acceptance/bundle/exec/cwd/script b/acceptance/bundle/exec/cwd/unix/script similarity index 100% rename from acceptance/bundle/exec/cwd/script rename to acceptance/bundle/exec/cwd/unix/script diff --git a/acceptance/bundle/exec/cwd/unix/test.toml b/acceptance/bundle/exec/cwd/unix/test.toml new file mode 100644 index 0000000000..7f30bdcde2 --- /dev/null +++ b/acceptance/bundle/exec/cwd/unix/test.toml @@ -0,0 +1,4 @@ +[GOOS] +windows = false +linux = true +darwin = true diff --git a/acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep b/acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/exec/cwd/windows/databricks.yml b/acceptance/bundle/exec/cwd/windows/databricks.yml new file mode 100644 index 0000000000..432311dab0 --- /dev/null +++ b/acceptance/bundle/exec/cwd/windows/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: foobar diff --git a/acceptance/bundle/exec/cwd/windows/output.txt b/acceptance/bundle/exec/cwd/windows/output.txt new file mode 100644 index 0000000000..e6496afab8 --- /dev/null +++ b/acceptance/bundle/exec/cwd/windows/output.txt @@ -0,0 +1,8 @@ + +>>> cd a/b/c + +>>> pwd +[TMPDIR]/a/b/c + +>>> [CLI] bundle exec -- pwd +[TMPDIR] diff --git a/acceptance/bundle/exec/cwd/windows/script b/acceptance/bundle/exec/cwd/windows/script new file mode 100644 index 0000000000..3461f6afcd --- /dev/null +++ b/acceptance/bundle/exec/cwd/windows/script @@ -0,0 +1,6 @@ +trace cd a/b/c + +trace pwd + +# Scripts that bundle exec executes should run from the bundle root. +trace $CLI bundle exec -- dir diff --git a/acceptance/bundle/exec/cwd/windows/test.toml b/acceptance/bundle/exec/cwd/windows/test.toml new file mode 100644 index 0000000000..22f2c79f0b --- /dev/null +++ b/acceptance/bundle/exec/cwd/windows/test.toml @@ -0,0 +1,4 @@ +[GOOS] +windows = true +linux = false +darwin = false From 4ef33dc2b4b688f306184526f042008acc13a0c1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:09:26 +0100 Subject: [PATCH 37/77] Revert "split windows test" This reverts commit 2c773683bc3133d11fd0cf04c2ca5de0431db4a8. --- acceptance/bundle/exec/basic/test.toml | 3 --- acceptance/bundle/exec/cmd-not-found/test.toml | 3 --- acceptance/bundle/exec/cwd/{unix => }/a/b/c/.gitkeep | 0 acceptance/bundle/exec/cwd/{unix => }/databricks.yml | 0 acceptance/bundle/exec/cwd/{unix => }/output.txt | 0 acceptance/bundle/exec/cwd/{unix => }/script | 0 acceptance/bundle/exec/cwd/unix/test.toml | 4 ---- acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep | 0 acceptance/bundle/exec/cwd/windows/databricks.yml | 2 -- acceptance/bundle/exec/cwd/windows/output.txt | 8 -------- acceptance/bundle/exec/cwd/windows/script | 6 ------ acceptance/bundle/exec/cwd/windows/test.toml | 4 ---- 12 files changed, 30 deletions(-) delete mode 100644 acceptance/bundle/exec/basic/test.toml delete mode 100644 acceptance/bundle/exec/cmd-not-found/test.toml rename acceptance/bundle/exec/cwd/{unix => }/a/b/c/.gitkeep (100%) rename acceptance/bundle/exec/cwd/{unix => }/databricks.yml (100%) rename acceptance/bundle/exec/cwd/{unix => }/output.txt (100%) rename acceptance/bundle/exec/cwd/{unix => }/script (100%) delete mode 100644 acceptance/bundle/exec/cwd/unix/test.toml delete mode 100644 acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep delete mode 100644 acceptance/bundle/exec/cwd/windows/databricks.yml delete mode 100644 acceptance/bundle/exec/cwd/windows/output.txt delete mode 100644 acceptance/bundle/exec/cwd/windows/script delete mode 100644 acceptance/bundle/exec/cwd/windows/test.toml diff --git a/acceptance/bundle/exec/basic/test.toml b/acceptance/bundle/exec/basic/test.toml deleted file mode 100644 index ca7d069ae6..0000000000 --- a/acceptance/bundle/exec/basic/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -[[Repls]] -Old = "%PATH%" -New = "$PATH" diff --git a/acceptance/bundle/exec/cmd-not-found/test.toml b/acceptance/bundle/exec/cmd-not-found/test.toml deleted file mode 100644 index ca7d069ae6..0000000000 --- a/acceptance/bundle/exec/cmd-not-found/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -[[Repls]] -Old = "%PATH%" -New = "$PATH" diff --git a/acceptance/bundle/exec/cwd/unix/a/b/c/.gitkeep b/acceptance/bundle/exec/cwd/a/b/c/.gitkeep similarity index 100% rename from acceptance/bundle/exec/cwd/unix/a/b/c/.gitkeep rename to acceptance/bundle/exec/cwd/a/b/c/.gitkeep diff --git a/acceptance/bundle/exec/cwd/unix/databricks.yml b/acceptance/bundle/exec/cwd/databricks.yml similarity index 100% rename from acceptance/bundle/exec/cwd/unix/databricks.yml rename to acceptance/bundle/exec/cwd/databricks.yml diff --git a/acceptance/bundle/exec/cwd/unix/output.txt b/acceptance/bundle/exec/cwd/output.txt similarity index 100% rename from acceptance/bundle/exec/cwd/unix/output.txt rename to acceptance/bundle/exec/cwd/output.txt diff --git a/acceptance/bundle/exec/cwd/unix/script b/acceptance/bundle/exec/cwd/script similarity index 100% rename from acceptance/bundle/exec/cwd/unix/script rename to acceptance/bundle/exec/cwd/script diff --git a/acceptance/bundle/exec/cwd/unix/test.toml b/acceptance/bundle/exec/cwd/unix/test.toml deleted file mode 100644 index 7f30bdcde2..0000000000 --- a/acceptance/bundle/exec/cwd/unix/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -[GOOS] -windows = false -linux = true -darwin = true diff --git a/acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep b/acceptance/bundle/exec/cwd/windows/a/b/c/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/acceptance/bundle/exec/cwd/windows/databricks.yml b/acceptance/bundle/exec/cwd/windows/databricks.yml deleted file mode 100644 index 432311dab0..0000000000 --- a/acceptance/bundle/exec/cwd/windows/databricks.yml +++ /dev/null @@ -1,2 +0,0 @@ -bundle: - name: foobar diff --git a/acceptance/bundle/exec/cwd/windows/output.txt b/acceptance/bundle/exec/cwd/windows/output.txt deleted file mode 100644 index e6496afab8..0000000000 --- a/acceptance/bundle/exec/cwd/windows/output.txt +++ /dev/null @@ -1,8 +0,0 @@ - ->>> cd a/b/c - ->>> pwd -[TMPDIR]/a/b/c - ->>> [CLI] bundle exec -- pwd -[TMPDIR] diff --git a/acceptance/bundle/exec/cwd/windows/script b/acceptance/bundle/exec/cwd/windows/script deleted file mode 100644 index 3461f6afcd..0000000000 --- a/acceptance/bundle/exec/cwd/windows/script +++ /dev/null @@ -1,6 +0,0 @@ -trace cd a/b/c - -trace pwd - -# Scripts that bundle exec executes should run from the bundle root. -trace $CLI bundle exec -- dir diff --git a/acceptance/bundle/exec/cwd/windows/test.toml b/acceptance/bundle/exec/cwd/windows/test.toml deleted file mode 100644 index 22f2c79f0b..0000000000 --- a/acceptance/bundle/exec/cwd/windows/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -[GOOS] -windows = true -linux = false -darwin = false From 996b58a1350ee9b5defb30cfc04c0264e1e84da3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:12:05 +0100 Subject: [PATCH 38/77] fix pwd --- acceptance/bundle/exec/basic/output.txt | 3 --- acceptance/bundle/exec/basic/script | 2 -- acceptance/bundle/exec/cwd/output.txt | 4 ++-- acceptance/bundle/exec/cwd/script | 4 ++-- acceptance/bundle/exec/no-auth/output.txt | 2 +- acceptance/bundle/exec/no-auth/script | 2 +- 6 files changed, 6 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index c9a651aed0..4c8668232b 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -2,9 +2,6 @@ >>> [CLI] bundle exec -- echo hello, world hello, world ->>> [CLI] bundle exec -- pwd -[TMPDIR] - >>> [CLI] bundle exec -- --help Error: starting "--help" failed: exec: "--help": executable file not found in $PATH diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 9d1a281c2b..285023492a 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -1,7 +1,5 @@ trace $CLI bundle exec -- echo "hello, world" -trace $CLI bundle exec -- pwd - # The CLI should not parse the --help flag and should try to run it as an executable # instead. trace $CLI bundle exec -- --help diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/exec/cwd/output.txt index e6496afab8..928bea82d8 100644 --- a/acceptance/bundle/exec/cwd/output.txt +++ b/acceptance/bundle/exec/cwd/output.txt @@ -1,8 +1,8 @@ >>> cd a/b/c ->>> pwd +>>> python3 -c import os; print(os.getcwd()) [TMPDIR]/a/b/c ->>> [CLI] bundle exec -- pwd +>>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) [TMPDIR] diff --git a/acceptance/bundle/exec/cwd/script b/acceptance/bundle/exec/cwd/script index 73a0966d3f..f5e866d06a 100644 --- a/acceptance/bundle/exec/cwd/script +++ b/acceptance/bundle/exec/cwd/script @@ -1,6 +1,6 @@ trace cd a/b/c -trace pwd +trace python3 -c 'import os; print(os.getcwd())' # Scripts that bundle exec executes should run from the bundle root. -trace $CLI bundle exec -- pwd +trace $CLI bundle exec -- python3 -c 'import os; print(os.getcwd())' diff --git a/acceptance/bundle/exec/no-auth/output.txt b/acceptance/bundle/exec/no-auth/output.txt index fce8e3f613..be94dae361 100644 --- a/acceptance/bundle/exec/no-auth/output.txt +++ b/acceptance/bundle/exec/no-auth/output.txt @@ -2,5 +2,5 @@ >>> [CLI] bundle exec -- echo hello hello ->>> [CLI] bundle exec -- pwd +>>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) [TMPDIR] diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/exec/no-auth/script index a1758ce997..c6b9c6bfe9 100644 --- a/acceptance/bundle/exec/no-auth/script +++ b/acceptance/bundle/exec/no-auth/script @@ -4,4 +4,4 @@ unset DATABRICKS_TOKEN # Confirm that bundle exec works for commands that do not require authentication, # even if authentication is not provided. trace $CLI bundle exec -- echo hello -trace $CLI bundle exec -- pwd +trace $CLI bundle exec -- python3 -c 'import os; print(os.getcwd())' From fcc2966118bf9aa50aa251c06d3b38aee07bcd54 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:14:08 +0100 Subject: [PATCH 39/77] - --- acceptance/bundle/exec/basic/output.txt | 8 ++++++++ acceptance/bundle/exec/basic/script | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 4c8668232b..6056a8c9e7 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -6,3 +6,11 @@ hello, world Error: starting "--help" failed: exec: "--help": executable file not found in $PATH Exit code: 1 + +>>> [CLI] bundle exec -- bash -c exit 5 +Error: running "bash -c exit 5" failed with exit code: 5 + +Exit code: 1 + +>>> [CLI] bundle exec -- bash -c echo hello > /dev/stderr +hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 285023492a..20b56ad1ae 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -2,7 +2,7 @@ trace $CLI bundle exec -- echo "hello, world" # The CLI should not parse the --help flag and should try to run it as an executable # instead. -trace $CLI bundle exec -- --help +errcode trace $CLI bundle exec -- --help # The error message should include the exit code. errcode trace $CLI bundle exec -- bash -c "exit 5" From 09c05db3b18f94fa38c3440ea2d7176626bd361b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:16:40 +0100 Subject: [PATCH 40/77] - --- acceptance/bundle/exec/basic/test.toml | 3 +++ acceptance/bundle/exec/cmd-not-found/test.toml | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 acceptance/bundle/exec/basic/test.toml create mode 100644 acceptance/bundle/exec/cmd-not-found/test.toml diff --git a/acceptance/bundle/exec/basic/test.toml b/acceptance/bundle/exec/basic/test.toml new file mode 100644 index 0000000000..ca7d069ae6 --- /dev/null +++ b/acceptance/bundle/exec/basic/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = "%PATH%" +New = "$PATH" diff --git a/acceptance/bundle/exec/cmd-not-found/test.toml b/acceptance/bundle/exec/cmd-not-found/test.toml new file mode 100644 index 0000000000..ca7d069ae6 --- /dev/null +++ b/acceptance/bundle/exec/cmd-not-found/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = "%PATH%" +New = "$PATH" From acff901127a9a592919d73fa2e19e060244ee83d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 11:39:09 +0100 Subject: [PATCH 41/77] fix windows --- acceptance/bundle/exec/basic/output.txt | 2 +- acceptance/bundle/exec/basic/test.toml | 6 +++++- acceptance/bundle/exec/cmd-not-found/output.txt | 2 +- acceptance/bundle/exec/cmd-not-found/script | 2 +- acceptance/bundle/exec/cmd-not-found/test.toml | 6 +++++- acceptance/bundle/exec/cwd/test.toml | 3 +++ cmd/bundle/exec.go | 2 +- 7 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 acceptance/bundle/exec/cwd/test.toml diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 6056a8c9e7..ef56f139fc 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -3,7 +3,7 @@ hello, world >>> [CLI] bundle exec -- --help -Error: starting "--help" failed: exec: "--help": executable file not found in $PATH +Error: starting "--help" failed: exec: "--help": executable file not found in PATH Exit code: 1 diff --git a/acceptance/bundle/exec/basic/test.toml b/acceptance/bundle/exec/basic/test.toml index ca7d069ae6..81f2393ae4 100644 --- a/acceptance/bundle/exec/basic/test.toml +++ b/acceptance/bundle/exec/basic/test.toml @@ -1,3 +1,7 @@ +[[Repls]] +Old = "\\$PATH" +New = "PATH" + [[Repls]] Old = "%PATH%" -New = "$PATH" +New = "PATH" diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/exec/cmd-not-found/output.txt index 8344948b44..e6284234e0 100644 --- a/acceptance/bundle/exec/cmd-not-found/output.txt +++ b/acceptance/bundle/exec/cmd-not-found/output.txt @@ -1,5 +1,5 @@ >>> [CLI] bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 -Error: starting "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in $PATH +Error: starting "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in PATH Exit code: 1 diff --git a/acceptance/bundle/exec/cmd-not-found/script b/acceptance/bundle/exec/cmd-not-found/script index 4a5d442f3a..0cefaea0cd 100644 --- a/acceptance/bundle/exec/cmd-not-found/script +++ b/acceptance/bundle/exec/cmd-not-found/script @@ -1 +1 @@ -trace $CLI bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 +errcode trace $CLI bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 diff --git a/acceptance/bundle/exec/cmd-not-found/test.toml b/acceptance/bundle/exec/cmd-not-found/test.toml index ca7d069ae6..81f2393ae4 100644 --- a/acceptance/bundle/exec/cmd-not-found/test.toml +++ b/acceptance/bundle/exec/cmd-not-found/test.toml @@ -1,3 +1,7 @@ +[[Repls]] +Old = "\\$PATH" +New = "PATH" + [[Repls]] Old = "%PATH%" -New = "$PATH" +New = "PATH" diff --git a/acceptance/bundle/exec/cwd/test.toml b/acceptance/bundle/exec/cwd/test.toml new file mode 100644 index 0000000000..a298217f21 --- /dev/null +++ b/acceptance/bundle/exec/cwd/test.toml @@ -0,0 +1,3 @@ +[[Repls]] +Old = '\\' +New = '/' diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index bbb8038b95..87cc9b929e 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -55,7 +55,7 @@ Example usage: // // This is only useful for when the Databricks CLI is the child command, // since if we do not explicitly pass the profile, the CLI will use the - // profile configured in the bundle YAML configuration (if any). + // auth configured in the bundle YAML configuration (if any). if b.Config.Workspace.Profile != "" { env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } From 7dc24b6484c3d6cd2827bd6e9be52d7111f61ed9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 12:56:38 +0100 Subject: [PATCH 42/77] move tests --- .../from_bundle/.databrickscfg | 3 +++ .../{ => from_bundle}/databricks.yml | 0 .../from_bundle/out.requests.txt | 9 +++++++ .../profile-is-passed/from_bundle/output.txt | 6 +++++ .../{ => from_bundle}/script | 4 ---- .../{ => from_flag}/.databrickscfg | 0 .../from_flag/databricks.yml | 5 ++++ .../{ => from_flag}/out.requests.txt | 7 +----- .../{ => from_flag}/output.txt | 6 ----- .../profile-is-passed/from_flag/script | 10 ++++++++ .../target-is-passed/default/.databrickscfg | 8 +++++++ .../{ => default}/databricks.yml | 4 +++- .../target-is-passed/default/out.requests.txt | 9 +++++++ .../target-is-passed/default/output.txt | 6 +++++ .../target-is-passed/default/script | 10 ++++++++ .../target-is-passed/from_flag/.databrickscfg | 8 +++++++ .../target-is-passed/from_flag/databricks.yml | 12 ++++++++++ .../{ => from_flag}/out.requests.txt | 16 +------------ .../target-is-passed/from_flag/output.txt | 6 +++++ .../target-is-passed/from_flag/script | 10 ++++++++ .../target-is-passed/output.txt | 24 ------------------- .../databricks-cli/target-is-passed/script | 15 ------------ 22 files changed, 107 insertions(+), 71 deletions(-) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg rename acceptance/bundle/exec/databricks-cli/profile-is-passed/{ => from_bundle}/databricks.yml (100%) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt rename acceptance/bundle/exec/databricks-cli/profile-is-passed/{ => from_bundle}/script (68%) rename acceptance/bundle/exec/databricks-cli/profile-is-passed/{ => from_flag}/.databrickscfg (100%) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml rename acceptance/bundle/exec/databricks-cli/profile-is-passed/{ => from_flag}/out.requests.txt (78%) rename acceptance/bundle/exec/databricks-cli/profile-is-passed/{ => from_flag}/output.txt (54%) create mode 100644 acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg rename acceptance/bundle/exec/databricks-cli/target-is-passed/{ => default}/databricks.yml (53%) create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/default/script create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml rename acceptance/bundle/exec/databricks-cli/target-is-passed/{ => from_flag}/out.requests.txt (61%) create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt create mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script delete mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt delete mode 100644 acceptance/bundle/exec/databricks-cli/target-is-passed/script diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg new file mode 100644 index 0000000000..405e344a02 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg @@ -0,0 +1,3 @@ +[someprofile] +host = $DATABRICKS_HOST +token = $DATABRICKS_TOKEN diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/databricks.yml rename to acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt new file mode 100644 index 0000000000..c4a1034826 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt @@ -0,0 +1,9 @@ +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt new file mode 100644 index 0000000000..8ca3fe8c56 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle exec -- [CLI] current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script similarity index 68% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/script rename to acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script index 33e07494c6..a614758e5e 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script @@ -1,5 +1,4 @@ # Replace placeholder with an actual host URL -envsubst < databricks.yml > out.yml && mv out.yml databricks.yml envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg @@ -9,6 +8,3 @@ unset DATABRICKS_TOKEN # This should use the profile configured in the bundle, i.e. PAT auth trace $CLI bundle exec -- $CLI current-user me - -# This should use myprofile, which uses oauth. -trace $CLI bundle exec --profile myprofile -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/.databrickscfg similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/.databrickscfg rename to acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml new file mode 100644 index 0000000000..5de7d1d968 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: foobar + +workspace: + profile: someprofile diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/out.requests.txt similarity index 78% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt rename to acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/out.requests.txt index 2b214d4fd2..bbad66cf8b 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/out.requests.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/out.requests.txt @@ -1,11 +1,6 @@ { - "headers": { - "Authorization": [ - "Bearer [DATABRICKS_TOKEN]" - ] - }, "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" + "path": "/oidc/.well-known/oauth-authorization-server" } { "method": "GET", diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt similarity index 54% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt rename to acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt index 06cdd1c440..30463a78fe 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt @@ -1,10 +1,4 @@ ->>> [CLI] bundle exec -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} - >>> [CLI] bundle exec --profile myprofile -- [CLI] current-user me { "id":"[USERID]", diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script new file mode 100644 index 0000000000..7d0124c0a6 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script @@ -0,0 +1,10 @@ +# Replace placeholder with an actual host URL +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Credentials will be picked up from .databrickscfg. Unset existing credentials. +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN + +# This should use myprofile, which uses oauth. +trace $CLI bundle exec --profile myprofile -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg new file mode 100644 index 0000000000..dbf302e9ce --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg @@ -0,0 +1,8 @@ +[pat-profile] +host = $DATABRICKS_HOST +token = $DATABRICKS_TOKEN + +[oauth-profile] +host = $DATABRICKS_HOST +client_id = client_id +client_secret = client_secret diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/databricks.yml b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/databricks.yml similarity index 53% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/databricks.yml rename to acceptance/bundle/exec/databricks-cli/target-is-passed/default/databricks.yml index 6b782c3257..7f808c24b1 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/databricks.yml +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/databricks.yml @@ -4,7 +4,9 @@ bundle: targets: pat: default: true + workspace: + profile: pat-profile oauth: workspace: - client_id: client_id + profile: oauth-profile diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt new file mode 100644 index 0000000000..c4a1034826 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt @@ -0,0 +1,9 @@ +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt new file mode 100644 index 0000000000..8ca3fe8c56 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle exec -- [CLI] current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script new file mode 100644 index 0000000000..4b036659d4 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script @@ -0,0 +1,10 @@ +# Replace placeholder with an actual host URL +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Credentials will be picked up from .databrickscfg. Unset existing credentials. +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN + +# Should use default target, which is bat based authentication +trace $CLI bundle exec -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg new file mode 100644 index 0000000000..dbf302e9ce --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg @@ -0,0 +1,8 @@ +[pat-profile] +host = $DATABRICKS_HOST +token = $DATABRICKS_TOKEN + +[oauth-profile] +host = $DATABRICKS_HOST +client_id = client_id +client_secret = client_secret diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml new file mode 100644 index 0000000000..7f808c24b1 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: foobar + +targets: + pat: + default: true + workspace: + profile: pat-profile + + oauth: + workspace: + profile: oauth-profile diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/out.requests.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/out.requests.txt similarity index 61% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/out.requests.txt rename to acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/out.requests.txt index 26efbbb08d..bbad66cf8b 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/out.requests.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/out.requests.txt @@ -1,20 +1,6 @@ { - "headers": { - "Authorization": [ - "Bearer [DATABRICKS_TOKEN]" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "headers": { - "Authorization": [ - "Bearer [DATABRICKS_TOKEN]" - ] - }, "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" + "path": "/oidc/.well-known/oauth-authorization-server" } { "method": "GET", diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt new file mode 100644 index 0000000000..828868f106 --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle exec -t oauth -- [CLI] current-user me +{ + "id":"[USERID]", + "userName":"[USERNAME]" +} diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script new file mode 100644 index 0000000000..58a407a10f --- /dev/null +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script @@ -0,0 +1,10 @@ +# Replace placeholder with an actual host URL +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Credentials will be picked up from .databrickscfg. Unset existing credentials. +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN + +# Explicitly select oauth target +trace $CLI bundle exec -t oauth -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt deleted file mode 100644 index 0751ac7dc0..0000000000 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/output.txt +++ /dev/null @@ -1,24 +0,0 @@ - ->>> [CLI] bundle exec -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} - ->>> [CLI] bundle exec -t pat -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} - ->>> errcode [CLI] bundle exec -t pat -- [CLI] current-user me -t oauth -Error: cannot resolve bundle auth configuration: validate: more than one authorization method configured: oauth and pat. Config: host=[DATABRICKS_URL], token=***, client_id=client_id, databricks_cli_path=[CLI]. Env: DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_CLI_PATH -Error: running "[CLI] current-user me -t oauth" failed with exit code: 1 - -Exit code: 1 - ->>> [CLI] bundle exec -t oauth -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/script deleted file mode 100644 index 238c36ae4b..0000000000 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/script +++ /dev/null @@ -1,15 +0,0 @@ -# Default target -trace $CLI bundle exec -- $CLI current-user me - -# Explicitly select default target -trace $CLI bundle exec -t pat -- $CLI current-user me - -# Conflicting targets selected. This should fail because for the child command -# pat would be configured via environment variables and oauth via the CLI resulting -# in more than one authorization method configured. -trace errcode $CLI bundle exec -t pat -- $CLI current-user me -t oauth - -# Explicitly select oauth target -unset DATABRICKS_TOKEN -export DATABRICKS_CLIENT_SECRET="client_secret" -trace $CLI bundle exec -t oauth -- $CLI current-user me From 90a95846ad0d013feead02103c70e7b06f2e9f09 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 13:00:06 +0100 Subject: [PATCH 43/77] - --- .../exec/databricks-cli/profile-is-passed/from_bundle/script | 2 +- .../exec/databricks-cli/profile-is-passed/from_flag/script | 2 +- .../exec/databricks-cli/target-is-passed/default/script | 4 ++-- .../exec/databricks-cli/target-is-passed/from_flag/script | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script index a614758e5e..e894e45820 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script @@ -1,4 +1,4 @@ -# Replace placeholder with an actual host URL +# Replace placeholder with an actual host URL and token envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script index 7d0124c0a6..9afda563b0 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script @@ -1,4 +1,4 @@ -# Replace placeholder with an actual host URL +# Replace placeholder with an actual host URL and token envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script index 4b036659d4..f719893cca 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script @@ -1,4 +1,4 @@ -# Replace placeholder with an actual host URL +# Replace placeholder with an actual host URL and token envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg @@ -6,5 +6,5 @@ export DATABRICKS_CONFIG_FILE=.databrickscfg unset DATABRICKS_HOST unset DATABRICKS_TOKEN -# Should use default target, which is bat based authentication +# Should use default target, which is pat based authentication trace $CLI bundle exec -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script index 58a407a10f..a314d02230 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script @@ -1,4 +1,4 @@ -# Replace placeholder with an actual host URL +# Replace placeholder with an actual host URL and token envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg From 8c094db50303477bde548139bf438ee7d9b89f7b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Mar 2025 13:17:09 +0100 Subject: [PATCH 44/77] general stderr --- acceptance/bundle/exec/basic/output.txt | 4 ++-- acceptance/bundle/exec/basic/script | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index ef56f139fc..1b932e9b20 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -12,5 +12,5 @@ Error: running "bash -c exit 5" failed with exit code: 5 Exit code: 1 ->>> [CLI] bundle exec -- bash -c echo hello > /dev/stderr -hello +>>> [CLI] bundle exec -- python3 -c import sys; print('Hello', file=sys.stderr) +Hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 20b56ad1ae..1aaf140dab 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -8,4 +8,4 @@ errcode trace $CLI bundle exec -- --help errcode trace $CLI bundle exec -- bash -c "exit 5" # stderr should also be passed through. -trace $CLI bundle exec -- bash -c "echo hello > /dev/stderr" +trace $CLI bundle exec -- python3 -c "import sys; print('Hello', file=sys.stderr)" From 5586121239cddfe58c21466e3185cb613aa42670 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Mar 2025 12:48:25 +0100 Subject: [PATCH 45/77] read target from flag --- cmd/bundle/exec.go | 9 +++------ cmd/root/bundle.go | 6 +++--- cmd/root/bundle_test.go | 6 +++--- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 87cc9b929e..d781ee4937 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -7,7 +7,6 @@ import ( "strings" "sync" - "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/auth" "github.com/spf13/cobra" @@ -42,13 +41,11 @@ Example usage: env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) - // If user has specified a target, pass it to the child command. DABs - // defines a "default" target which is a placeholder for when no target is defined. - // If that's the case, i.e. no targets are defined, then do not pass the target. + // If user has specified a target, pass it to the child command. // // This is only useful for when the Databricks CLI is the child command. - if b.Config.Bundle.Target != mutator.DefaultTargetPlaceholder { - env = append(env, "DATABRICKS_BUNDLE_TARGET="+b.Config.Bundle.Target) + if target := root.GetTarget(cmd); target != "" { + env = append(env, "DATABRICKS_CONFIG_TARGET="+target) } // If the bundle has a profile configured, explicitly pass it to the child command. diff --git a/cmd/root/bundle.go b/cmd/root/bundle.go index b408037079..85d47431d5 100644 --- a/cmd/root/bundle.go +++ b/cmd/root/bundle.go @@ -12,8 +12,8 @@ import ( "golang.org/x/exp/maps" ) -// getTarget returns the name of the target to operate in. -func getTarget(cmd *cobra.Command) (value string) { +// GetTarget returns the name of the target to operate in. +func GetTarget(cmd *cobra.Command) (value string) { target, isFlagSet := targetFlagValue(cmd) if isFlagSet { return target @@ -77,7 +77,7 @@ func configureBundle(cmd *cobra.Command, b *bundle.Bundle) (*bundle.Bundle, diag // Load bundle and select target. ctx := cmd.Context() var diags diag.Diagnostics - if target := getTarget(cmd); target == "" { + if target := GetTarget(cmd); target == "" { diags = phases.LoadDefaultTarget(ctx, b) } else { diags = phases.LoadNamedTarget(ctx, b, target) diff --git a/cmd/root/bundle_test.go b/cmd/root/bundle_test.go index 3517b02e46..bf7c44188c 100644 --- a/cmd/root/bundle_test.go +++ b/cmd/root/bundle_test.go @@ -215,7 +215,7 @@ func TestTargetFlagFull(t *testing.T) { err := cmd.ExecuteContext(ctx) assert.NoError(t, err) - assert.Equal(t, "development", getTarget(cmd)) + assert.Equal(t, "development", GetTarget(cmd)) } func TestTargetFlagShort(t *testing.T) { @@ -227,7 +227,7 @@ func TestTargetFlagShort(t *testing.T) { err := cmd.ExecuteContext(ctx) assert.NoError(t, err) - assert.Equal(t, "production", getTarget(cmd)) + assert.Equal(t, "production", GetTarget(cmd)) } // TODO: remove when environment flag is fully deprecated @@ -241,5 +241,5 @@ func TestTargetEnvironmentFlag(t *testing.T) { err := cmd.ExecuteContext(ctx) assert.NoError(t, err) - assert.Equal(t, "development", getTarget(cmd)) + assert.Equal(t, "development", GetTarget(cmd)) } From 26684b192e15fa6d397798483eefd3cd15d1f781 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Mar 2025 12:50:38 +0100 Subject: [PATCH 46/77] :- --- cmd/auth/describe.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/auth/describe.go b/cmd/auth/describe.go index faaf64f8fd..cf278db941 100644 --- a/cmd/auth/describe.go +++ b/cmd/auth/describe.go @@ -57,7 +57,7 @@ func newDescribeCommand() *cobra.Command { var err error status, err = getAuthStatus(cmd, args, showSensitive, func(cmd *cobra.Command, args []string) (*config.Config, bool, error) { isAccount, err := root.MustAnyClient(cmd, args) - return root.ConfigUsed(cmd.Context()), isAccount, err + return command.ConfigUsed(cmd.Context()), isAccount, err }) if err != nil { return err From d9b5f5e18565257a3ac9655e0d12c2a794c07347 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Mar 2025 12:51:01 +0100 Subject: [PATCH 47/77] fix build --- cmd/bundle/exec.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index d781ee4937..023ea33958 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/command" "github.com/spf13/cobra" ) @@ -39,7 +40,7 @@ Example usage: childCmd := exec.Command(args[0], args[1:]...) - env := auth.ProcessEnv(root.ConfigUsed(cmd.Context())) + env := auth.ProcessEnv(command.ConfigUsed(cmd.Context())) // If user has specified a target, pass it to the child command. // From 987220b9de6321fdbe70244a0e337341540a7dd4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Mar 2025 12:54:20 +0100 Subject: [PATCH 48/77] update docs --- cmd/bundle/exec.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 023ea33958..3a04d6a984 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -20,9 +20,12 @@ func newExecCommand() *cobra.Command { Args: cobra.MinimumNArgs(1), Long: `Execute a command using the same authentication context as the bundle -The current working directory of the provided command will be set to the root +Note: The current working directory of the provided command will be set to the root of the bundle. +Authentication to the input command will be provided by setting the appropriate +environment variables that Databricks tools use to authenticate. + Example usage: 1. databricks bundle exec -- echo "hello, world" 2. databricks bundle exec -- /bin/bash -c "echo hello" From 98f33a80b8b52deaedf6bb4610149093fee0536e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 14:28:45 +0200 Subject: [PATCH 49/77] - --- cmd/bundle/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index b7683a676c..574ad10163 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -46,7 +46,7 @@ func resolveRunArgument(ctx context.Context, b *bundle.Bundle, args []string) (s return "", nil, err } return key, args, nil - } a + } if len(args) < 1 { return "", nil, errors.New("expected a KEY of the resource to run") From 9ea61443603ff4ed771f0deb3aa72f7b2b814791 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 16:19:46 +0200 Subject: [PATCH 50/77] use execve --- acceptance/bundle/exec/basic/output.txt | 5 +- .../bundle/exec/cmd-not-found/output.txt | 2 +- acceptance/bundle/exec/cwd/output.txt | 4 +- acceptance/bundle/exec/no-auth/output.txt | 2 +- cmd/bundle/exec.go | 97 +++---------------- libs/exec/execv.go | 22 +++++ libs/exec/execv_unix.go | 24 +++++ libs/exec/execv_windows.go | 95 ++++++++++++++++++ 8 files changed, 158 insertions(+), 93 deletions(-) create mode 100644 libs/exec/execv.go create mode 100644 libs/exec/execv_unix.go create mode 100644 libs/exec/execv_windows.go diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 1b932e9b20..9cb5f8c7b8 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -3,14 +3,13 @@ hello, world >>> [CLI] bundle exec -- --help -Error: starting "--help" failed: exec: "--help": executable file not found in PATH +Error: running "--help" failed: looking up "--help" in PATH failed: exec: "--help": executable file not found in PATH Exit code: 1 >>> [CLI] bundle exec -- bash -c exit 5 -Error: running "bash -c exit 5" failed with exit code: 5 -Exit code: 1 +Exit code: 5 >>> [CLI] bundle exec -- python3 -c import sys; print('Hello', file=sys.stderr) Hello diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/exec/cmd-not-found/output.txt index e6284234e0..73bc70e6c6 100644 --- a/acceptance/bundle/exec/cmd-not-found/output.txt +++ b/acceptance/bundle/exec/cmd-not-found/output.txt @@ -1,5 +1,5 @@ >>> [CLI] bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 -Error: starting "doesnotexist arg1 arg2 --flag1 --flag2" failed: exec: "doesnotexist": executable file not found in PATH +Error: running "doesnotexist arg1 arg2 --flag1 --flag2" failed: looking up "doesnotexist" in PATH failed: exec: "doesnotexist": executable file not found in PATH Exit code: 1 diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/exec/cwd/output.txt index 928bea82d8..9276774605 100644 --- a/acceptance/bundle/exec/cwd/output.txt +++ b/acceptance/bundle/exec/cwd/output.txt @@ -2,7 +2,7 @@ >>> cd a/b/c >>> python3 -c import os; print(os.getcwd()) -[TMPDIR]/a/b/c +[TEST_TMP_DIR]/a/b/c >>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) -[TMPDIR] +[TEST_TMP_DIR] diff --git a/acceptance/bundle/exec/no-auth/output.txt b/acceptance/bundle/exec/no-auth/output.txt index be94dae361..436991346b 100644 --- a/acceptance/bundle/exec/no-auth/output.txt +++ b/acceptance/bundle/exec/no-auth/output.txt @@ -3,4 +3,4 @@ hello >>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) -[TMPDIR] +[TEST_TMP_DIR] diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index 3a04d6a984..f6473cc2eb 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -1,15 +1,13 @@ package bundle import ( - "bufio" "fmt" - "os/exec" "strings" - "sync" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/auth" - "github.com/databricks/cli/libs/command" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/exec" "github.com/spf13/cobra" ) @@ -41,9 +39,7 @@ Example usage: return diags.Error() } - childCmd := exec.Command(args[0], args[1:]...) - - env := auth.ProcessEnv(command.ConfigUsed(cmd.Context())) + env := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) // If user has specified a target, pass it to the child command. // @@ -61,86 +57,15 @@ Example usage: env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } - childCmd.Env = env - - // Execute all scripts from the bundle root directory. This behavior can - // be surprising in isolation, but we do it to keep the behavior consistent - // for both these cases: - // 1. One shot commands like `databricks bundle exec -- echo hello` - // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. - // - // TODO(shreyas): Add a DATABRICKS_BUNDLE_INITIAL_CWD environment variable - // that users can read to figure out the original CWD. I'll do that when - // adding support for the scripts section. - childCmd.Dir = b.BundleRootPath - - stdout, err := childCmd.StdoutPipe() - if err != nil { - return fmt.Errorf("creating stdout pipe failed: %w", err) - } - - stderr, err := childCmd.StderrPipe() - if err != nil { - return fmt.Errorf("creating stderr pipe failed: %w", err) - } - - // Start the child command. - err = childCmd.Start() - if err != nil { - return fmt.Errorf("starting %q failed: %w", strings.Join(args, " "), err) - } + // TODO: Test singal propogation in the acceptance testS? - var wg sync.WaitGroup - wg.Add(2) - - var stdoutErr error - go func() { - defer wg.Done() - - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - _, err = fmt.Fprintln(cmd.OutOrStdout(), scanner.Text()) - if err != nil { - stdoutErr = err - break - } - } - }() - - var stderrErr error - go func() { - defer wg.Done() - - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - _, err = fmt.Fprintln(cmd.ErrOrStderr(), scanner.Text()) - if err != nil { - stderrErr = err - break - } - } - }() - - wg.Wait() - - if stdoutErr != nil { - return fmt.Errorf("writing stdout failed: %w", stdoutErr) - } - - if stderrErr != nil { - return fmt.Errorf("writing stderr failed: %w", stderrErr) - } - - err = childCmd.Wait() - if exitErr, ok := err.(*exec.ExitError); ok { - // We don't make the parent CLI process exit with the same exit code - // as the child process because the exit codes for the CLI have not - // been standardized yet. - // - // This keeps the door open for us to associate specific exit codes - // with specific classes of errors in the future. - return fmt.Errorf("running %q failed with exit code: %d", strings.Join(args, " "), exitErr.ExitCode()) - } + err := exec.Execv(exec.ExecvOptions{ + Args: args, + Env: env, + Dir: b.BundleRootPath, + Stderr: cmd.ErrOrStderr(), + Stdout: cmd.OutOrStdout(), + }) if err != nil { return fmt.Errorf("running %q failed: %w", strings.Join(args, " "), err) } diff --git a/libs/exec/execv.go b/libs/exec/execv.go new file mode 100644 index 0000000000..c477d76dd4 --- /dev/null +++ b/libs/exec/execv.go @@ -0,0 +1,22 @@ +package exec + +import "io" + +// TODO(shreyas): Add a DATABRICKS_BUNDLE_INITIAL_CWD environment variable +// that users can read to figure out the original CWD. I'll do that when +// adding support for the scripts section. +type ExecvOptions struct { + Args []string + Env []string + Dir string + + // Stderr and Stdout are only used for Windows where execv is not supported. + // For Unix, the Stdout and Stderr are automatically inherited during the exec + // system call. + Stderr io.Writer + Stdout io.Writer +} + +func Execv(opts ExecvOptions) error { + return execv(opts) +} diff --git a/libs/exec/execv_unix.go b/libs/exec/execv_unix.go new file mode 100644 index 0000000000..663cc1360c --- /dev/null +++ b/libs/exec/execv_unix.go @@ -0,0 +1,24 @@ +//go:build linux || darwin + +package exec + +import ( + "fmt" + "os" + "os/exec" + "syscall" +) + +func execv(opts ExecvOptions) error { + err := os.Chdir(opts.Dir) + if err != nil { + return fmt.Errorf("changing directory to %s failed: %w", opts.Dir, err) + } + + // execve syscall does not perform PATH lookup and + path, err := exec.LookPath(opts.Args[0]) + if err != nil { + return fmt.Errorf("looking up %q in PATH failed: %w", opts.Args[0], err) + } + return syscall.Exec(path, opts.Args, opts.Env) +} diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go new file mode 100644 index 0000000000..04be6edf98 --- /dev/null +++ b/libs/exec/execv_windows.go @@ -0,0 +1,95 @@ +//go:build windows + +package exec + +import ( + "bufio" + "fmt" + "os" + "os/exec" + "strings" + "sync" +) + +// Note: Windows does not support an execv syscall that replaces the current process. +// Thus we emulate that by launching a child process and streaming the output and returning +// the exit code. +// ref: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/execv-wexecv?view=msvc-170 +func execv(opts ExecvOptions) error { + cmd := exec.Command(opts.Name, opts.Args...) + + // TODO: Move this comment. + // Execute all scripts from the bundle root directory. This behavior can + // be surprising in isolation, but we do it to keep the behavior consistent + // for both these cases: + // 1. One shot commands like `databricks bundle exec -- echo hello` + // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. + cmd.Dir = opts.Dir + + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("creating stdout pipe failed: %w", err) + } + + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("creating stderr pipe failed: %w", err) + } + + // Start the child command. + err = cmd.Start() + if err != nil { + return fmt.Errorf("starting %s %s failed: %w", opts.Name, strings.Join(opts.Args, " "), err) + } + + var wg sync.WaitGroup + wg.Add(2) + + var stdoutErr error + go func() { + defer wg.Done() + + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + _, err = fmt.Fprintln(opts.Stdout, scanner.Text()) + if err != nil { + stdoutErr = err + break + } + } + }() + + var stderrErr error + go func() { + defer wg.Done() + + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + _, err = fmt.Fprintln(opts.Stderr, scanner.Text()) + if err != nil { + stderrErr = err + break + } + } + }() + + wg.Wait() + + if stdoutErr != nil { + return fmt.Errorf("writing stdout failed: %w", stdoutErr) + } + + if stderrErr != nil { + return fmt.Errorf("writing stderr failed: %w", stderrErr) + } + + err = cmd.Wait() + if exitErr, ok := err.(*exec.ExitError); ok { + os.Exit(exitErr.ExitCode()) + } + if err != nil { + return fmt.Errorf("running %s %s failed: %w", opts.Name, strings.Join(opts.Args, " "), err) + } + + return nil +} From 3c4ff15e007ce05c253c5ba5293ec6c38bb206c2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:31:10 +0200 Subject: [PATCH 51/77] switch over to run --- acceptance/bundle/exec/basic/output.txt | 10 ++--- acceptance/bundle/exec/basic/script | 8 ++-- .../bundle/exec/cmd-not-found/output.txt | 4 +- acceptance/bundle/exec/cmd-not-found/script | 2 +- acceptance/bundle/exec/cwd/output.txt | 2 +- acceptance/bundle/exec/cwd/script | 4 +- .../profile-is-passed/from_bundle/output.txt | 2 +- .../profile-is-passed/from_bundle/script | 2 +- .../profile-is-passed/from_flag/output.txt | 2 +- .../profile-is-passed/from_flag/script | 2 +- .../target-is-passed/default/output.txt | 2 +- .../target-is-passed/default/script | 2 +- .../target-is-passed/from_flag/output.txt | 2 +- .../target-is-passed/from_flag/script | 2 +- acceptance/bundle/exec/no-auth/output.txt | 4 +- acceptance/bundle/exec/no-auth/script | 6 +-- acceptance/bundle/exec/no-bundle/output.txt | 2 +- acceptance/bundle/exec/no-bundle/script | 2 +- .../bundle/exec/no-separator/output.txt | 4 +- acceptance/bundle/exec/no-separator/script | 2 +- cmd/bundle/exec.go | 1 - cmd/bundle/run.go | 39 +++++++++++++++++++ 22 files changed, 72 insertions(+), 34 deletions(-) diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/exec/basic/output.txt index 9cb5f8c7b8..f941077c9d 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/exec/basic/output.txt @@ -1,15 +1,15 @@ ->>> [CLI] bundle exec -- echo hello, world +>>> [CLI] bundle run -- echo hello, world hello, world ->>> [CLI] bundle exec -- --help -Error: running "--help" failed: looking up "--help" in PATH failed: exec: "--help": executable file not found in PATH +>>> [CLI] bundle run -- --help +Error: looking up "--help" in PATH failed: exec: "--help": executable file not found in PATH Exit code: 1 ->>> [CLI] bundle exec -- bash -c exit 5 +>>> [CLI] bundle run -- bash -c exit 5 Exit code: 5 ->>> [CLI] bundle exec -- python3 -c import sys; print('Hello', file=sys.stderr) +>>> [CLI] bundle run -- python3 -c import sys; print('Hello', file=sys.stderr) Hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/exec/basic/script index 1aaf140dab..8bcb80cc41 100644 --- a/acceptance/bundle/exec/basic/script +++ b/acceptance/bundle/exec/basic/script @@ -1,11 +1,11 @@ -trace $CLI bundle exec -- echo "hello, world" +trace $CLI bundle run -- echo "hello, world" # The CLI should not parse the --help flag and should try to run it as an executable # instead. -errcode trace $CLI bundle exec -- --help +errcode trace $CLI bundle run -- --help # The error message should include the exit code. -errcode trace $CLI bundle exec -- bash -c "exit 5" +errcode trace $CLI bundle run -- bash -c "exit 5" # stderr should also be passed through. -trace $CLI bundle exec -- python3 -c "import sys; print('Hello', file=sys.stderr)" +trace $CLI bundle run -- python3 -c "import sys; print('Hello', file=sys.stderr)" diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/exec/cmd-not-found/output.txt index 73bc70e6c6..275f67bf71 100644 --- a/acceptance/bundle/exec/cmd-not-found/output.txt +++ b/acceptance/bundle/exec/cmd-not-found/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 -Error: running "doesnotexist arg1 arg2 --flag1 --flag2" failed: looking up "doesnotexist" in PATH failed: exec: "doesnotexist": executable file not found in PATH +>>> [CLI] bundle run -- doesnotexist arg1 arg2 --flag1 --flag2 +Error: looking up "doesnotexist" in PATH failed: exec: "doesnotexist": executable file not found in PATH Exit code: 1 diff --git a/acceptance/bundle/exec/cmd-not-found/script b/acceptance/bundle/exec/cmd-not-found/script index 0cefaea0cd..78919d576e 100644 --- a/acceptance/bundle/exec/cmd-not-found/script +++ b/acceptance/bundle/exec/cmd-not-found/script @@ -1 +1 @@ -errcode trace $CLI bundle exec -- doesnotexist arg1 arg2 --flag1 --flag2 +errcode trace $CLI bundle run -- doesnotexist arg1 arg2 --flag1 --flag2 diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/exec/cwd/output.txt index 9276774605..8a571d5de8 100644 --- a/acceptance/bundle/exec/cwd/output.txt +++ b/acceptance/bundle/exec/cwd/output.txt @@ -4,5 +4,5 @@ >>> python3 -c import os; print(os.getcwd()) [TEST_TMP_DIR]/a/b/c ->>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) +>>> [CLI] bundle run -- python3 -c import os; print(os.getcwd()) [TEST_TMP_DIR] diff --git a/acceptance/bundle/exec/cwd/script b/acceptance/bundle/exec/cwd/script index f5e866d06a..b106425244 100644 --- a/acceptance/bundle/exec/cwd/script +++ b/acceptance/bundle/exec/cwd/script @@ -2,5 +2,5 @@ trace cd a/b/c trace python3 -c 'import os; print(os.getcwd())' -# Scripts that bundle exec executes should run from the bundle root. -trace $CLI bundle exec -- python3 -c 'import os; print(os.getcwd())' +# Scripts that bundle run executes should run from the bundle root. +trace $CLI bundle run -- python3 -c 'import os; print(os.getcwd())' diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt index 8ca3fe8c56..704a6ac915 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec -- [CLI] current-user me +>>> [CLI] bundle run -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script index e894e45820..28bc0f1b84 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script @@ -7,4 +7,4 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # This should use the profile configured in the bundle, i.e. PAT auth -trace $CLI bundle exec -- $CLI current-user me +trace $CLI bundle run -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt index 30463a78fe..e2c200ce1d 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec --profile myprofile -- [CLI] current-user me +>>> [CLI] bundle run --profile myprofile -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script index 9afda563b0..41049c25f4 100644 --- a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script +++ b/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script @@ -7,4 +7,4 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # This should use myprofile, which uses oauth. -trace $CLI bundle exec --profile myprofile -- $CLI current-user me +trace $CLI bundle run --profile myprofile -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt index 8ca3fe8c56..704a6ac915 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec -- [CLI] current-user me +>>> [CLI] bundle run -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script index f719893cca..f9d3610b4e 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script @@ -7,4 +7,4 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # Should use default target, which is pat based authentication -trace $CLI bundle exec -- $CLI current-user me +trace $CLI bundle run -- $CLI current-user me diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt index 828868f106..9d568529fd 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec -t oauth -- [CLI] current-user me +>>> [CLI] bundle run -t oauth -- [CLI] current-user me { "id":"[USERID]", "userName":"[USERNAME]" diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script index a314d02230..fcae775022 100644 --- a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script +++ b/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script @@ -7,4 +7,4 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN # Explicitly select oauth target -trace $CLI bundle exec -t oauth -- $CLI current-user me +trace $CLI bundle run -t oauth -- $CLI current-user me diff --git a/acceptance/bundle/exec/no-auth/output.txt b/acceptance/bundle/exec/no-auth/output.txt index 436991346b..6257073282 100644 --- a/acceptance/bundle/exec/no-auth/output.txt +++ b/acceptance/bundle/exec/no-auth/output.txt @@ -1,6 +1,6 @@ ->>> [CLI] bundle exec -- echo hello +>>> [CLI] bundle run -- echo hello hello ->>> [CLI] bundle exec -- python3 -c import os; print(os.getcwd()) +>>> [CLI] bundle run -- python3 -c import os; print(os.getcwd()) [TEST_TMP_DIR] diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/exec/no-auth/script index c6b9c6bfe9..f96de1cf0b 100644 --- a/acceptance/bundle/exec/no-auth/script +++ b/acceptance/bundle/exec/no-auth/script @@ -1,7 +1,7 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN -# Confirm that bundle exec works for commands that do not require authentication, +# Confirm that bundle run works for commands that do not require authentication, # even if authentication is not provided. -trace $CLI bundle exec -- echo hello -trace $CLI bundle exec -- python3 -c 'import os; print(os.getcwd())' +trace $CLI bundle run -- echo hello +trace $CLI bundle run -- python3 -c 'import os; print(os.getcwd())' diff --git a/acceptance/bundle/exec/no-bundle/output.txt b/acceptance/bundle/exec/no-bundle/output.txt index 7030c5405c..f961335f9d 100644 --- a/acceptance/bundle/exec/no-bundle/output.txt +++ b/acceptance/bundle/exec/no-bundle/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec -- echo hello +>>> [CLI] bundle run -- echo hello Error: unable to locate bundle root: databricks.yml not found Exit code: 1 diff --git a/acceptance/bundle/exec/no-bundle/script b/acceptance/bundle/exec/no-bundle/script index b872f65db1..925f1c19cd 100644 --- a/acceptance/bundle/exec/no-bundle/script +++ b/acceptance/bundle/exec/no-bundle/script @@ -1 +1 @@ -trace $CLI bundle exec -- echo hello +trace $CLI bundle run -- echo hello diff --git a/acceptance/bundle/exec/no-separator/output.txt b/acceptance/bundle/exec/no-separator/output.txt index ef7b6f1ce1..693c5d324a 100644 --- a/acceptance/bundle/exec/no-separator/output.txt +++ b/acceptance/bundle/exec/no-separator/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle exec echo hello -Error: Please add a '--' separator. Usage: 'databricks bundle exec -- echo hello' +>>> [CLI] bundle run echo hello +Error: unable to locate bundle root: databricks.yml not found Exit code: 1 diff --git a/acceptance/bundle/exec/no-separator/script b/acceptance/bundle/exec/no-separator/script index 050538fa4d..6a0dfbf29a 100644 --- a/acceptance/bundle/exec/no-separator/script +++ b/acceptance/bundle/exec/no-separator/script @@ -1 +1 @@ -trace $CLI bundle exec echo hello +trace $CLI bundle run echo hello diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index f6473cc2eb..c0f5f804f0 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -57,7 +57,6 @@ Example usage: env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } - // TODO: Test singal propogation in the acceptance testS? err := exec.Execv(exec.ExecvOptions{ Args: args, diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 574ad10163..3fed411e8e 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -14,7 +14,10 @@ import ( "github.com/databricks/cli/bundle/run/output" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/exec" "github.com/databricks/cli/libs/flags" "github.com/spf13/cobra" "golang.org/x/exp/maps" @@ -111,6 +114,13 @@ task or a Python wheel task, the second example applies. return diags.Error() } + // If user runs the bundle run command as: + // databricks bundle run -- + // we execute the command inline. + if cmd.ArgsLenAtDash() == 0 { + return executeInline(cmd, args, b) + } + diags = phases.Initialize(ctx, b) if err := diags.Error(); err != nil { return err @@ -209,3 +219,32 @@ task or a Python wheel task, the second example applies. return cmd } + +// TODO: Test singal propogation in the acceptance testS? +func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { + env := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) + + // If user has specified a target, pass it to the child command. + // + // This is only useful for when the Databricks CLI is the child command. + if target := root.GetTarget(cmd); target != "" { + env = append(env, "DATABRICKS_CONFIG_TARGET="+target) + } + + // If the bundle has a profile configured, explicitly pass it to the child command. + // + // This is only useful for when the Databricks CLI is the child command, + // since if we do not explicitly pass the profile, the CLI will use the + // auth configured in the bundle YAML configuration (if any). + if b.Config.Workspace.Profile != "" { + env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) + } + + return exec.Execv(exec.ExecvOptions{ + Args: args, + Env: env, + Dir: b.BundleRootPath, + Stderr: cmd.ErrOrStderr(), + Stdout: cmd.OutOrStdout(), + }) +} From 2866956ba74adb286443502fd27360646bbc07ac Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:43:48 +0200 Subject: [PATCH 52/77] -gs --- libs/exec/{execv_windows.go => exec_windows.go} | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) rename libs/exec/{execv_windows.go => exec_windows.go} (82%) diff --git a/libs/exec/execv_windows.go b/libs/exec/exec_windows.go similarity index 82% rename from libs/exec/execv_windows.go rename to libs/exec/exec_windows.go index 04be6edf98..15a29d842d 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/exec_windows.go @@ -1,5 +1,4 @@ //go:build windows - package exec import ( @@ -16,7 +15,13 @@ import ( // the exit code. // ref: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/execv-wexecv?view=msvc-170 func execv(opts ExecvOptions) error { - cmd := exec.Command(opts.Name, opts.Args...) + path, err := exec.LookPath(opts.Args[0]) + if err != nil { + return fmt.Errorf("looking up %q in PATH failed: %w", opts.Args[0], err) + } + + // TODO: Validate atleast one arg before calling execv. + cmd := exec.Command(path, opts.Args[1:]...) // TODO: Move this comment. // Execute all scripts from the bundle root directory. This behavior can @@ -25,6 +30,7 @@ func execv(opts ExecvOptions) error { // 1. One shot commands like `databricks bundle exec -- echo hello` // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. cmd.Dir = opts.Dir + cmd.Env = opts.Env stdout, err := cmd.StdoutPipe() if err != nil { @@ -39,7 +45,7 @@ func execv(opts ExecvOptions) error { // Start the child command. err = cmd.Start() if err != nil { - return fmt.Errorf("starting %s %s failed: %w", opts.Name, strings.Join(opts.Args, " "), err) + return fmt.Errorf(" %s failed: %w", strings.Join(opts.Args, " "), err) } var wg sync.WaitGroup @@ -88,7 +94,7 @@ func execv(opts ExecvOptions) error { os.Exit(exitErr.ExitCode()) } if err != nil { - return fmt.Errorf("running %s %s failed: %w", opts.Name, strings.Join(opts.Args, " "), err) + return fmt.Errorf("running %s failed: %w", strings.Join(opts.Args, " "), err) } return nil From 6239f4b952366aeb2ef2b81d546ac1528b6515f1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:43:56 +0200 Subject: [PATCH 53/77] - --- libs/exec/{exec_windows.go => execv_windows.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename libs/exec/{exec_windows.go => execv_windows.go} (100%) diff --git a/libs/exec/exec_windows.go b/libs/exec/execv_windows.go similarity index 100% rename from libs/exec/exec_windows.go rename to libs/exec/execv_windows.go From 28ea87e1639f004f5ee8a1e3a8a7c84e25bd32c5 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:48:58 +0200 Subject: [PATCH 54/77] - --- .../bundle/{exec => run/inline-script}/basic/databricks.yml | 0 .../bundle/{exec => run/inline-script}/basic/output.txt | 4 +++- acceptance/bundle/{exec => run/inline-script}/basic/script | 0 acceptance/bundle/{exec => run/inline-script}/basic/test.toml | 0 .../{exec => run/inline-script}/cmd-not-found/databricks.yml | 0 .../{exec => run/inline-script}/cmd-not-found/output.txt | 0 .../bundle/{exec => run/inline-script}/cmd-not-found/script | 0 .../{exec => run/inline-script}/cmd-not-found/test.toml | 0 .../bundle/{exec => run/inline-script}/cwd/a/b/c/.gitkeep | 0 .../bundle/{exec => run/inline-script}/cwd/databricks.yml | 0 acceptance/bundle/{exec => run/inline-script}/cwd/output.txt | 0 acceptance/bundle/{exec => run/inline-script}/cwd/script | 0 acceptance/bundle/{exec => run/inline-script}/cwd/test.toml | 0 .../profile-is-passed/from_bundle/.databrickscfg | 0 .../profile-is-passed/from_bundle/databricks.yml | 0 .../profile-is-passed/from_bundle/out.requests.txt | 0 .../databricks-cli/profile-is-passed/from_bundle/output.txt | 0 .../databricks-cli/profile-is-passed/from_bundle/script | 0 .../databricks-cli/profile-is-passed/from_flag/.databrickscfg | 0 .../databricks-cli/profile-is-passed/from_flag/databricks.yml | 0 .../profile-is-passed/from_flag/out.requests.txt | 0 .../databricks-cli/profile-is-passed/from_flag/output.txt | 0 .../databricks-cli/profile-is-passed/from_flag/script | 0 .../databricks-cli/target-is-passed/default/.databrickscfg | 0 .../databricks-cli/target-is-passed/default/databricks.yml | 0 .../databricks-cli/target-is-passed/default/out.requests.txt | 0 .../databricks-cli/target-is-passed/default/output.txt | 0 .../databricks-cli/target-is-passed/default/script | 0 .../databricks-cli/target-is-passed/from_flag/.databrickscfg | 0 .../databricks-cli/target-is-passed/from_flag/databricks.yml | 0 .../target-is-passed/from_flag/out.requests.txt | 0 .../databricks-cli/target-is-passed/from_flag/output.txt | 0 .../databricks-cli/target-is-passed/from_flag/script | 0 .../{exec => run/inline-script}/databricks-cli/test.toml | 0 .../bundle/{exec => run/inline-script}/no-auth/databricks.yml | 0 .../bundle/{exec => run/inline-script}/no-auth/output.txt | 0 acceptance/bundle/{exec => run/inline-script}/no-auth/script | 0 .../bundle/{exec => run/inline-script}/no-bundle/output.txt | 0 .../bundle/{exec => run/inline-script}/no-bundle/script | 0 .../{exec => run/inline-script}/no-separator/output.txt | 0 .../bundle/{exec => run/inline-script}/no-separator/script | 0 acceptance/bundle/{exec => run/inline-script}/test.toml | 0 cmd/bundle/run.go | 2 +- 43 files changed, 4 insertions(+), 2 deletions(-) rename acceptance/bundle/{exec => run/inline-script}/basic/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/basic/output.txt (71%) rename acceptance/bundle/{exec => run/inline-script}/basic/script (100%) rename acceptance/bundle/{exec => run/inline-script}/basic/test.toml (100%) rename acceptance/bundle/{exec => run/inline-script}/cmd-not-found/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/cmd-not-found/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/cmd-not-found/script (100%) rename acceptance/bundle/{exec => run/inline-script}/cmd-not-found/test.toml (100%) rename acceptance/bundle/{exec => run/inline-script}/cwd/a/b/c/.gitkeep (100%) rename acceptance/bundle/{exec => run/inline-script}/cwd/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/cwd/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/cwd/script (100%) rename acceptance/bundle/{exec => run/inline-script}/cwd/test.toml (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_bundle/.databrickscfg (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_bundle/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_bundle/out.requests.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_bundle/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_bundle/script (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_flag/.databrickscfg (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_flag/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_flag/out.requests.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_flag/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/profile-is-passed/from_flag/script (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/default/.databrickscfg (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/default/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/default/out.requests.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/default/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/default/script (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/from_flag/.databrickscfg (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/from_flag/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/from_flag/out.requests.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/from_flag/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/target-is-passed/from_flag/script (100%) rename acceptance/bundle/{exec => run/inline-script}/databricks-cli/test.toml (100%) rename acceptance/bundle/{exec => run/inline-script}/no-auth/databricks.yml (100%) rename acceptance/bundle/{exec => run/inline-script}/no-auth/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/no-auth/script (100%) rename acceptance/bundle/{exec => run/inline-script}/no-bundle/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/no-bundle/script (100%) rename acceptance/bundle/{exec => run/inline-script}/no-separator/output.txt (100%) rename acceptance/bundle/{exec => run/inline-script}/no-separator/script (100%) rename acceptance/bundle/{exec => run/inline-script}/test.toml (100%) diff --git a/acceptance/bundle/exec/basic/databricks.yml b/acceptance/bundle/run/inline-script/basic/databricks.yml similarity index 100% rename from acceptance/bundle/exec/basic/databricks.yml rename to acceptance/bundle/run/inline-script/basic/databricks.yml diff --git a/acceptance/bundle/exec/basic/output.txt b/acceptance/bundle/run/inline-script/basic/output.txt similarity index 71% rename from acceptance/bundle/exec/basic/output.txt rename to acceptance/bundle/run/inline-script/basic/output.txt index f941077c9d..b26b3f3232 100644 --- a/acceptance/bundle/exec/basic/output.txt +++ b/acceptance/bundle/run/inline-script/basic/output.txt @@ -11,5 +11,7 @@ Exit code: 1 Exit code: 5 ->>> [CLI] bundle run -- python3 -c import sys; print('Hello', file=sys.stderr) +>>> cat stderr.txt + +>>> [CLI] bundle exec -- python3 -c import sys; print('Hello', file=sys.stderr) Hello diff --git a/acceptance/bundle/exec/basic/script b/acceptance/bundle/run/inline-script/basic/script similarity index 100% rename from acceptance/bundle/exec/basic/script rename to acceptance/bundle/run/inline-script/basic/script diff --git a/acceptance/bundle/exec/basic/test.toml b/acceptance/bundle/run/inline-script/basic/test.toml similarity index 100% rename from acceptance/bundle/exec/basic/test.toml rename to acceptance/bundle/run/inline-script/basic/test.toml diff --git a/acceptance/bundle/exec/cmd-not-found/databricks.yml b/acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml similarity index 100% rename from acceptance/bundle/exec/cmd-not-found/databricks.yml rename to acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml diff --git a/acceptance/bundle/exec/cmd-not-found/output.txt b/acceptance/bundle/run/inline-script/cmd-not-found/output.txt similarity index 100% rename from acceptance/bundle/exec/cmd-not-found/output.txt rename to acceptance/bundle/run/inline-script/cmd-not-found/output.txt diff --git a/acceptance/bundle/exec/cmd-not-found/script b/acceptance/bundle/run/inline-script/cmd-not-found/script similarity index 100% rename from acceptance/bundle/exec/cmd-not-found/script rename to acceptance/bundle/run/inline-script/cmd-not-found/script diff --git a/acceptance/bundle/exec/cmd-not-found/test.toml b/acceptance/bundle/run/inline-script/cmd-not-found/test.toml similarity index 100% rename from acceptance/bundle/exec/cmd-not-found/test.toml rename to acceptance/bundle/run/inline-script/cmd-not-found/test.toml diff --git a/acceptance/bundle/exec/cwd/a/b/c/.gitkeep b/acceptance/bundle/run/inline-script/cwd/a/b/c/.gitkeep similarity index 100% rename from acceptance/bundle/exec/cwd/a/b/c/.gitkeep rename to acceptance/bundle/run/inline-script/cwd/a/b/c/.gitkeep diff --git a/acceptance/bundle/exec/cwd/databricks.yml b/acceptance/bundle/run/inline-script/cwd/databricks.yml similarity index 100% rename from acceptance/bundle/exec/cwd/databricks.yml rename to acceptance/bundle/run/inline-script/cwd/databricks.yml diff --git a/acceptance/bundle/exec/cwd/output.txt b/acceptance/bundle/run/inline-script/cwd/output.txt similarity index 100% rename from acceptance/bundle/exec/cwd/output.txt rename to acceptance/bundle/run/inline-script/cwd/output.txt diff --git a/acceptance/bundle/exec/cwd/script b/acceptance/bundle/run/inline-script/cwd/script similarity index 100% rename from acceptance/bundle/exec/cwd/script rename to acceptance/bundle/run/inline-script/cwd/script diff --git a/acceptance/bundle/exec/cwd/test.toml b/acceptance/bundle/run/inline-script/cwd/test.toml similarity index 100% rename from acceptance/bundle/exec/cwd/test.toml rename to acceptance/bundle/run/inline-script/cwd/test.toml diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/.databrickscfg rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/databricks.yml b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/databricks.yml rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/out.requests.txt rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/output.txt rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_bundle/script rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/.databrickscfg b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/.databrickscfg similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/.databrickscfg rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/databricks.yml rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.requests.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/out.requests.txt rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/output.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/output.txt rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/output.txt diff --git a/acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/script similarity index 100% rename from acceptance/bundle/exec/databricks-cli/profile-is-passed/from_flag/script rename to acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/script diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/.databrickscfg similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/default/.databrickscfg rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/databricks.yml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/default/databricks.yml rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/default/out.requests.txt rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/default/output.txt rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/default/script b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/script similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/default/script rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/script diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/.databrickscfg similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/.databrickscfg rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/.databrickscfg diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/databricks.yml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/databricks.yml rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/databricks.yml diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/out.requests.txt rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/output.txt rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt diff --git a/acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/script similarity index 100% rename from acceptance/bundle/exec/databricks-cli/target-is-passed/from_flag/script rename to acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/script diff --git a/acceptance/bundle/exec/databricks-cli/test.toml b/acceptance/bundle/run/inline-script/databricks-cli/test.toml similarity index 100% rename from acceptance/bundle/exec/databricks-cli/test.toml rename to acceptance/bundle/run/inline-script/databricks-cli/test.toml diff --git a/acceptance/bundle/exec/no-auth/databricks.yml b/acceptance/bundle/run/inline-script/no-auth/databricks.yml similarity index 100% rename from acceptance/bundle/exec/no-auth/databricks.yml rename to acceptance/bundle/run/inline-script/no-auth/databricks.yml diff --git a/acceptance/bundle/exec/no-auth/output.txt b/acceptance/bundle/run/inline-script/no-auth/output.txt similarity index 100% rename from acceptance/bundle/exec/no-auth/output.txt rename to acceptance/bundle/run/inline-script/no-auth/output.txt diff --git a/acceptance/bundle/exec/no-auth/script b/acceptance/bundle/run/inline-script/no-auth/script similarity index 100% rename from acceptance/bundle/exec/no-auth/script rename to acceptance/bundle/run/inline-script/no-auth/script diff --git a/acceptance/bundle/exec/no-bundle/output.txt b/acceptance/bundle/run/inline-script/no-bundle/output.txt similarity index 100% rename from acceptance/bundle/exec/no-bundle/output.txt rename to acceptance/bundle/run/inline-script/no-bundle/output.txt diff --git a/acceptance/bundle/exec/no-bundle/script b/acceptance/bundle/run/inline-script/no-bundle/script similarity index 100% rename from acceptance/bundle/exec/no-bundle/script rename to acceptance/bundle/run/inline-script/no-bundle/script diff --git a/acceptance/bundle/exec/no-separator/output.txt b/acceptance/bundle/run/inline-script/no-separator/output.txt similarity index 100% rename from acceptance/bundle/exec/no-separator/output.txt rename to acceptance/bundle/run/inline-script/no-separator/output.txt diff --git a/acceptance/bundle/exec/no-separator/script b/acceptance/bundle/run/inline-script/no-separator/script similarity index 100% rename from acceptance/bundle/exec/no-separator/script rename to acceptance/bundle/run/inline-script/no-separator/script diff --git a/acceptance/bundle/exec/test.toml b/acceptance/bundle/run/inline-script/test.toml similarity index 100% rename from acceptance/bundle/exec/test.toml rename to acceptance/bundle/run/inline-script/test.toml diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 3fed411e8e..15c01c18c8 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -117,7 +117,7 @@ task or a Python wheel task, the second example applies. // If user runs the bundle run command as: // databricks bundle run -- // we execute the command inline. - if cmd.ArgsLenAtDash() == 0 { + if cmd.ArgsLenAtDash() == 0 && len(args) > 0 { return executeInline(cmd, args, b) } From a6d161a1b0798609e1cee2b3178ee6e186a9ff9b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:49:27 +0200 Subject: [PATCH 55/77] - --- acceptance/bundle/run/inline-script/basic/script | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/run/inline-script/basic/script b/acceptance/bundle/run/inline-script/basic/script index 8bcb80cc41..350705035c 100644 --- a/acceptance/bundle/run/inline-script/basic/script +++ b/acceptance/bundle/run/inline-script/basic/script @@ -8,4 +8,6 @@ errcode trace $CLI bundle run -- --help errcode trace $CLI bundle run -- bash -c "exit 5" # stderr should also be passed through. -trace $CLI bundle run -- python3 -c "import sys; print('Hello', file=sys.stderr)" +trace $CLI bundle exec -- python3 -c "import sys; print('Hello', file=sys.stderr)" 2> stderr.txt +trace cat stderr.txt +rm stderr.txt From a7ffe5db820c32149565e24735f66ec48439fc8e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:51:44 +0200 Subject: [PATCH 56/77] - --- acceptance/bundle/run/inline-script/basic/output.txt | 2 +- .../bundle/run/inline-script/cmd-not-found/databricks.yml | 2 -- .../bundle/run/inline-script/cmd-not-found/output.txt | 5 ----- acceptance/bundle/run/inline-script/cmd-not-found/script | 1 - .../bundle/run/inline-script/cmd-not-found/test.toml | 7 ------- libs/exec/execv_unix.go | 2 +- libs/exec/execv_windows.go | 2 +- 7 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml delete mode 100644 acceptance/bundle/run/inline-script/cmd-not-found/output.txt delete mode 100644 acceptance/bundle/run/inline-script/cmd-not-found/script delete mode 100644 acceptance/bundle/run/inline-script/cmd-not-found/test.toml diff --git a/acceptance/bundle/run/inline-script/basic/output.txt b/acceptance/bundle/run/inline-script/basic/output.txt index b26b3f3232..70ffeb1f0c 100644 --- a/acceptance/bundle/run/inline-script/basic/output.txt +++ b/acceptance/bundle/run/inline-script/basic/output.txt @@ -3,7 +3,7 @@ hello, world >>> [CLI] bundle run -- --help -Error: looking up "--help" in PATH failed: exec: "--help": executable file not found in PATH +Error: looking up "--help" failed: exec: "--help": executable file not found in PATH Exit code: 1 diff --git a/acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml b/acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml deleted file mode 100644 index 432311dab0..0000000000 --- a/acceptance/bundle/run/inline-script/cmd-not-found/databricks.yml +++ /dev/null @@ -1,2 +0,0 @@ -bundle: - name: foobar diff --git a/acceptance/bundle/run/inline-script/cmd-not-found/output.txt b/acceptance/bundle/run/inline-script/cmd-not-found/output.txt deleted file mode 100644 index 275f67bf71..0000000000 --- a/acceptance/bundle/run/inline-script/cmd-not-found/output.txt +++ /dev/null @@ -1,5 +0,0 @@ - ->>> [CLI] bundle run -- doesnotexist arg1 arg2 --flag1 --flag2 -Error: looking up "doesnotexist" in PATH failed: exec: "doesnotexist": executable file not found in PATH - -Exit code: 1 diff --git a/acceptance/bundle/run/inline-script/cmd-not-found/script b/acceptance/bundle/run/inline-script/cmd-not-found/script deleted file mode 100644 index 78919d576e..0000000000 --- a/acceptance/bundle/run/inline-script/cmd-not-found/script +++ /dev/null @@ -1 +0,0 @@ -errcode trace $CLI bundle run -- doesnotexist arg1 arg2 --flag1 --flag2 diff --git a/acceptance/bundle/run/inline-script/cmd-not-found/test.toml b/acceptance/bundle/run/inline-script/cmd-not-found/test.toml deleted file mode 100644 index 81f2393ae4..0000000000 --- a/acceptance/bundle/run/inline-script/cmd-not-found/test.toml +++ /dev/null @@ -1,7 +0,0 @@ -[[Repls]] -Old = "\\$PATH" -New = "PATH" - -[[Repls]] -Old = "%PATH%" -New = "PATH" diff --git a/libs/exec/execv_unix.go b/libs/exec/execv_unix.go index 663cc1360c..bbb896ad03 100644 --- a/libs/exec/execv_unix.go +++ b/libs/exec/execv_unix.go @@ -18,7 +18,7 @@ func execv(opts ExecvOptions) error { // execve syscall does not perform PATH lookup and path, err := exec.LookPath(opts.Args[0]) if err != nil { - return fmt.Errorf("looking up %q in PATH failed: %w", opts.Args[0], err) + return fmt.Errorf("looking up %q failed: %w", opts.Args[0], err) } return syscall.Exec(path, opts.Args, opts.Env) } diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index 15a29d842d..a2738c67ca 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -17,7 +17,7 @@ import ( func execv(opts ExecvOptions) error { path, err := exec.LookPath(opts.Args[0]) if err != nil { - return fmt.Errorf("looking up %q in PATH failed: %w", opts.Args[0], err) + return fmt.Errorf("looking up %q failed: %w", opts.Args[0], err) } // TODO: Validate atleast one arg before calling execv. From c56f6f5da7d2508108f8dea99076e48c380f8da7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:57:37 +0200 Subject: [PATCH 57/77] - --- acceptance/bundle/run/inline-script/no-auth/output.txt | 5 +++++ acceptance/bundle/run/inline-script/no-auth/script | 1 + 2 files changed, 6 insertions(+) diff --git a/acceptance/bundle/run/inline-script/no-auth/output.txt b/acceptance/bundle/run/inline-script/no-auth/output.txt index 6257073282..1fab88ec25 100644 --- a/acceptance/bundle/run/inline-script/no-auth/output.txt +++ b/acceptance/bundle/run/inline-script/no-auth/output.txt @@ -4,3 +4,8 @@ hello >>> [CLI] bundle run -- python3 -c import os; print(os.getcwd()) [TEST_TMP_DIR] + +>>> [CLI] bundle run -- [CLI] current-user me +Error: default auth: cannot configure default credentials, please check https://docs.databricks.com/en/dev-tools/auth.html#databricks-client-unified-authentication to configure credentials for your preferred authentication method. Config: databricks_cli_path=[CLI]. Env: DATABRICKS_CLI_PATH + +Exit code: 1 diff --git a/acceptance/bundle/run/inline-script/no-auth/script b/acceptance/bundle/run/inline-script/no-auth/script index f96de1cf0b..8fec852ddc 100644 --- a/acceptance/bundle/run/inline-script/no-auth/script +++ b/acceptance/bundle/run/inline-script/no-auth/script @@ -5,3 +5,4 @@ unset DATABRICKS_TOKEN # even if authentication is not provided. trace $CLI bundle run -- echo hello trace $CLI bundle run -- python3 -c 'import os; print(os.getcwd())' +trace $CLI bundle run -- $CLI current-user me From a0fcc5d52e2c8940ff64a4f1d296fcc42d030d01 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 17:59:33 +0200 Subject: [PATCH 58/77] - --- bundle/config/mutator/default_target.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bundle/config/mutator/default_target.go b/bundle/config/mutator/default_target.go index c2e9e8acf2..73d99002a0 100644 --- a/bundle/config/mutator/default_target.go +++ b/bundle/config/mutator/default_target.go @@ -13,13 +13,11 @@ type defineDefaultTarget struct { name string } -const DefaultTargetPlaceholder = "default" - // DefineDefaultTarget adds a target named "default" // to the configuration if none have been defined. func DefineDefaultTarget() bundle.Mutator { return &defineDefaultTarget{ - name: DefaultTargetPlaceholder, + name: "default", } } From f30dffa1bf13ad7001e463ce5671e1deed1f6e95 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:02:01 +0200 Subject: [PATCH 59/77] - --- cmd/bundle/exec.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go index c0f5f804f0..fc95015763 100644 --- a/cmd/bundle/exec.go +++ b/cmd/bundle/exec.go @@ -57,7 +57,6 @@ Example usage: env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) } - err := exec.Execv(exec.ExecvOptions{ Args: args, Env: env, From 7d46902a1b0b74ea0929a2a6d2963991f536c614 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:34:54 +0200 Subject: [PATCH 60/77] - --- cmd/bundle/exec.go | 76 ---------------------------------------------- cmd/bundle/run.go | 19 ++++++++++-- 2 files changed, 17 insertions(+), 78 deletions(-) delete mode 100644 cmd/bundle/exec.go diff --git a/cmd/bundle/exec.go b/cmd/bundle/exec.go deleted file mode 100644 index fc95015763..0000000000 --- a/cmd/bundle/exec.go +++ /dev/null @@ -1,76 +0,0 @@ -package bundle - -import ( - "fmt" - "strings" - - "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/auth" - "github.com/databricks/cli/libs/cmdctx" - "github.com/databricks/cli/libs/exec" - "github.com/spf13/cobra" -) - -func newExecCommand() *cobra.Command { - execCmd := &cobra.Command{ - Use: "exec", - Short: "Execute a command using the same authentication context as the bundle", - Args: cobra.MinimumNArgs(1), - Long: `Execute a command using the same authentication context as the bundle - -Note: The current working directory of the provided command will be set to the root -of the bundle. - -Authentication to the input command will be provided by setting the appropriate -environment variables that Databricks tools use to authenticate. - -Example usage: -1. databricks bundle exec -- echo "hello, world" -2. databricks bundle exec -- /bin/bash -c "echo hello" -3. databricks bundle exec -- uv run pytest`, - RunE: func(cmd *cobra.Command, args []string) error { - if cmd.ArgsLenAtDash() != 0 { - return fmt.Errorf("Please add a '--' separator. Usage: 'databricks bundle exec -- %s'", strings.Join(args, " ")) - } - - // Load the bundle configuration to get the authentication credentials. - b, diags := root.MustConfigureBundle(cmd) - if diags.HasError() { - return diags.Error() - } - - env := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) - - // If user has specified a target, pass it to the child command. - // - // This is only useful for when the Databricks CLI is the child command. - if target := root.GetTarget(cmd); target != "" { - env = append(env, "DATABRICKS_CONFIG_TARGET="+target) - } - - // If the bundle has a profile configured, explicitly pass it to the child command. - // - // This is only useful for when the Databricks CLI is the child command, - // since if we do not explicitly pass the profile, the CLI will use the - // auth configured in the bundle YAML configuration (if any). - if b.Config.Workspace.Profile != "" { - env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) - } - - err := exec.Execv(exec.ExecvOptions{ - Args: args, - Env: env, - Dir: b.BundleRootPath, - Stderr: cmd.ErrOrStderr(), - Stdout: cmd.OutOrStdout(), - }) - if err != nil { - return fmt.Errorf("running %q failed: %w", strings.Join(args, " "), err) - } - - return nil - }, - } - - return execCmd -} diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 15c01c18c8..52c81affc8 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -76,7 +76,7 @@ func keyToRunner(b *bundle.Bundle, arg string) (run.Runner, error) { func newRunCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "run [flags] KEY", + Use: "run [flags] [KEY]", Short: "Run a job or pipeline update", Long: `Run the job or pipeline identified by KEY. @@ -96,6 +96,22 @@ parameter names. If the specified job does not use job parameters and the job has a Python file task or a Python wheel task, the second example applies. + +--------------------------------------------------------- + +You can also use the bundle run command to execute scripts / commands in the same +authentication context as the bundle. + +Note: The current working directory of the provided command will be set to the root +of the bundle. + +Authentication to the input command will be provided by setting the appropriate +environment variables that Databricks tools use to authenticate. + +Example usage: +1. databricks bundle exec -- echo "hello, world" +2. databricks bundle exec -- /bin/bash -c "echo hello" +3. databricks bundle exec -- uv run pytest `, } @@ -220,7 +236,6 @@ task or a Python wheel task, the second example applies. return cmd } -// TODO: Test singal propogation in the acceptance testS? func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { env := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) From c0c1ac65c8b44e47ba31666e3f720f40c45b2404 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:37:20 +0200 Subject: [PATCH 61/77] - --- cmd/bundle/bundle.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index e0818c2f97..fb88cd7d05 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -28,6 +28,5 @@ func New() *cobra.Command { cmd.AddCommand(newDebugCommand()) cmd.AddCommand(deployment.NewDeploymentCommand()) cmd.AddCommand(newOpenCommand()) - cmd.AddCommand(newExecCommand()) return cmd } From 3b579252683c1308fb37b7273a0c62b6c2c09f4c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:37:32 +0200 Subject: [PATCH 62/77] - --- libs/exec/execv_windows.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index a2738c67ca..ac9ce786db 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -1,4 +1,5 @@ //go:build windows + package exec import ( From e5e6ea4ea90b1da88e441b9dd0a1df81a6a1b944 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:38:55 +0200 Subject: [PATCH 63/77] cleanup --- cmd/bundle/run.go | 9 +++++++-- libs/exec/execv_windows.go | 7 ------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 52c81affc8..d8263068fd 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -256,8 +256,13 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { } return exec.Execv(exec.ExecvOptions{ - Args: args, - Env: env, + Args: args, + Env: env, + // Execute all scripts from the bundle root directory. This behavior can + // be surprising in isolation, but we do it to keep the behavior consistent + // for both these cases: + // 1. One shot commands like `databricks bundle exec -- echo hello` + // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. Dir: b.BundleRootPath, Stderr: cmd.ErrOrStderr(), Stdout: cmd.OutOrStdout(), diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index ac9ce786db..1fd525a8e4 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -21,15 +21,8 @@ func execv(opts ExecvOptions) error { return fmt.Errorf("looking up %q failed: %w", opts.Args[0], err) } - // TODO: Validate atleast one arg before calling execv. cmd := exec.Command(path, opts.Args[1:]...) - // TODO: Move this comment. - // Execute all scripts from the bundle root directory. This behavior can - // be surprising in isolation, but we do it to keep the behavior consistent - // for both these cases: - // 1. One shot commands like `databricks bundle exec -- echo hello` - // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. cmd.Dir = opts.Dir cmd.Env = opts.Env From b0bb768d4440408643b7bb2c94be42d36a9af49e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:41:36 +0200 Subject: [PATCH 64/77] - --- acceptance/bundle/help/bundle-run/output.txt | 18 +++++++++++++++++- acceptance/bundle/help/bundle/output.txt | 1 - .../bundle/run/inline-script/basic/output.txt | 2 +- .../bundle/run/inline-script/basic/script | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index 4b9efbf2ae..d86ad53740 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -19,8 +19,24 @@ parameter names. If the specified job does not use job parameters and the job has a Python file task or a Python wheel task, the second example applies. +--------------------------------------------------------- + +You can also use the bundle run command to execute scripts / commands in the same +authentication context as the bundle. + +Note: The current working directory of the provided command will be set to the root +of the bundle. + +Authentication to the input command will be provided by setting the appropriate +environment variables that Databricks tools use to authenticate. + +Example usage: +1. databricks bundle exec -- echo "hello, world" +2. databricks bundle exec -- /bin/bash -c "echo hello" +3. databricks bundle exec -- uv run pytest + Usage: - databricks bundle run [flags] KEY + databricks bundle run [flags] [KEY] Job Flags: --params stringToString comma separated k=v pairs for job parameters (default []) diff --git a/acceptance/bundle/help/bundle/output.txt b/acceptance/bundle/help/bundle/output.txt index bb885c80e5..fc6dd623dd 100644 --- a/acceptance/bundle/help/bundle/output.txt +++ b/acceptance/bundle/help/bundle/output.txt @@ -11,7 +11,6 @@ Available Commands: deploy Deploy bundle deployment Deployment related commands destroy Destroy deployed bundle resources - exec Execute a command using the same authentication context as the bundle generate Generate bundle configuration init Initialize using a bundle template open Open a resource in the browser diff --git a/acceptance/bundle/run/inline-script/basic/output.txt b/acceptance/bundle/run/inline-script/basic/output.txt index 70ffeb1f0c..294980e3c5 100644 --- a/acceptance/bundle/run/inline-script/basic/output.txt +++ b/acceptance/bundle/run/inline-script/basic/output.txt @@ -13,5 +13,5 @@ Exit code: 5 >>> cat stderr.txt ->>> [CLI] bundle exec -- python3 -c import sys; print('Hello', file=sys.stderr) +>>> [CLI] bundle run -- python3 -c import sys; print('Hello', file=sys.stderr) Hello diff --git a/acceptance/bundle/run/inline-script/basic/script b/acceptance/bundle/run/inline-script/basic/script index 350705035c..c1b5e70ec0 100644 --- a/acceptance/bundle/run/inline-script/basic/script +++ b/acceptance/bundle/run/inline-script/basic/script @@ -8,6 +8,6 @@ errcode trace $CLI bundle run -- --help errcode trace $CLI bundle run -- bash -c "exit 5" # stderr should also be passed through. -trace $CLI bundle exec -- python3 -c "import sys; print('Hello', file=sys.stderr)" 2> stderr.txt +trace $CLI bundle run -- python3 -c "import sys; print('Hello', file=sys.stderr)" 2> stderr.txt trace cat stderr.txt rm stderr.txt From fa6151bdb24737233f0c7ca825b9d5ffa354e5cb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:44:30 +0200 Subject: [PATCH 65/77] - --- acceptance/bundle/help/bundle-run/output.txt | 8 +++++--- cmd/bundle/run.go | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/help/bundle-run/output.txt b/acceptance/bundle/help/bundle-run/output.txt index d86ad53740..4f0a6b107c 100644 --- a/acceptance/bundle/help/bundle-run/output.txt +++ b/acceptance/bundle/help/bundle-run/output.txt @@ -31,9 +31,11 @@ Authentication to the input command will be provided by setting the appropriate environment variables that Databricks tools use to authenticate. Example usage: -1. databricks bundle exec -- echo "hello, world" -2. databricks bundle exec -- /bin/bash -c "echo hello" -3. databricks bundle exec -- uv run pytest +1. databricks bundle run -- echo "hello, world" +2. databricks bundle run -- /bin/bash -c "echo hello" +3. databricks bundle run -- uv run pytest + +--------------------------------------------------------- Usage: databricks bundle run [flags] [KEY] diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index d8263068fd..69e7a0f583 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -109,9 +109,11 @@ Authentication to the input command will be provided by setting the appropriate environment variables that Databricks tools use to authenticate. Example usage: -1. databricks bundle exec -- echo "hello, world" -2. databricks bundle exec -- /bin/bash -c "echo hello" -3. databricks bundle exec -- uv run pytest +1. databricks bundle run -- echo "hello, world" +2. databricks bundle run -- /bin/bash -c "echo hello" +3. databricks bundle run -- uv run pytest + +--------------------------------------------------------- `, } From 396fc2153fd6277bf39d3743699d90f8ff42fef7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:22:48 +0200 Subject: [PATCH 66/77] Add acceptance test for bundle run --- acceptance/bundle/run/basic/databricks.yml | 8 +++++ acceptance/bundle/run/basic/output.txt | 16 ++++++++++ acceptance/bundle/run/basic/script | 5 ++++ acceptance/internal/server.go | 34 ++++++++++++++++++++++ bundle/run/job.go | 2 +- libs/testserver/fake_workspace.go | 15 ++++++++++ 6 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/run/basic/databricks.yml create mode 100644 acceptance/bundle/run/basic/output.txt create mode 100644 acceptance/bundle/run/basic/script diff --git a/acceptance/bundle/run/basic/databricks.yml b/acceptance/bundle/run/basic/databricks.yml new file mode 100644 index 0000000000..5cbd3648b4 --- /dev/null +++ b/acceptance/bundle/run/basic/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: caterpillar + + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/run/basic/output.txt b/acceptance/bundle/run/basic/output.txt new file mode 100644 index 0000000000..92641a209d --- /dev/null +++ b/acceptance/bundle/run/basic/output.txt @@ -0,0 +1,16 @@ + +>>> [CLI] bundle run +Error: expected a KEY of the resource to run + +Exit code: 1 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/caterpillar/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run foo +Run URL: [DATABRICKS_URL]/job/run/1 + +2025-04-09 18:22:15 "foobar" TERMINATED diff --git a/acceptance/bundle/run/basic/script b/acceptance/bundle/run/basic/script new file mode 100644 index 0000000000..cc2c3d32fc --- /dev/null +++ b/acceptance/bundle/run/basic/script @@ -0,0 +1,5 @@ +errcode trace $CLI bundle run + +errcode trace $CLI bundle deploy + +errcode trace $CLI bundle run foo diff --git a/acceptance/internal/server.go b/acceptance/internal/server.go index 3156eeca74..f0f6878f4f 100644 --- a/acceptance/internal/server.go +++ b/acceptance/internal/server.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "strconv" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/iam" @@ -182,6 +183,39 @@ func AddHandlers(server *testserver.Server) { return req.Workspace.JobsList() }) + server.Handle("POST", "/api/2.2/jobs/run-now", func(req testserver.Request) any { + var request jobs.RunNow + if err := json.Unmarshal(req.Body, &request); err != nil { + return testserver.Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: 500, + } + } + + return req.Workspace.JobsRunNow(request.JobId) + }) + + server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { + runId := req.URL.Query().Get("run_id") + runIdInt, err := strconv.ParseInt(runId, 10, 64) + if err != nil { + return testserver.Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: 500, + } + } + + return jobs.Run{ + RunId: runIdInt, + State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + }, + RunPageUrl: fmt.Sprintf("%s/job/run/%d", server.URL, runIdInt), + RunType: jobs.RunTypeJobRun, + RunName: "foobar", + } + }) + server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(_ testserver.Request) any { return map[string]string{ "authorization_endpoint": server.URL + "oidc/v1/authorize", diff --git a/bundle/run/job.go b/bundle/run/job.go index 2489ca619d..3340948cb1 100644 --- a/bundle/run/job.go +++ b/bundle/run/job.go @@ -188,7 +188,7 @@ func (r *jobRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, e waiter, err := w.Jobs.RunNow(ctx, *req) if err != nil { - return nil, errors.New("cannot start job") + return nil, fmt.Errorf("cannot start job: %w", err) } if opts.NoWait { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index b2c98f9b4a..799937a924 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -192,6 +192,21 @@ func (s *FakeWorkspace) JobsGet(jobId string) Response { } } +func (s *FakeWorkspace) JobsRunNow(jobId int64) Response { + _, ok := s.jobs[jobId] + if !ok { + return Response{ + StatusCode: 404, + } + } + + return Response{ + Body: jobs.RunNowResponse{ + RunId: 1, + }, + } +} + func (s *FakeWorkspace) PipelinesGet(pipelineId string) Response { spec, ok := s.Pipelines[pipelineId] if !ok { From 8add0f8e3dbe1d47f7bbea9b6ce0036ad5bbd734 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:30:54 +0200 Subject: [PATCH 67/77] - --- acceptance/bundle/run/basic/databricks.yml | 10 ++++++++++ acceptance/bundle/run/basic/foo.py | 1 + acceptance/bundle/run/basic/output.txt | 20 +++++++++++++++++++- acceptance/bundle/run/basic/script | 11 +++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/run/basic/foo.py diff --git a/acceptance/bundle/run/basic/databricks.yml b/acceptance/bundle/run/basic/databricks.yml index 5cbd3648b4..c1761a8b97 100644 --- a/acceptance/bundle/run/basic/databricks.yml +++ b/acceptance/bundle/run/basic/databricks.yml @@ -6,3 +6,13 @@ resources: jobs: foo: name: foo + tasks: + - task_key: task + spark_python_task: + python_file: ./foo.py + environment_key: default + + environments: + - environment_key: default + spec: + client: "2" diff --git a/acceptance/bundle/run/basic/foo.py b/acceptance/bundle/run/basic/foo.py new file mode 100644 index 0000000000..b917a726c9 --- /dev/null +++ b/acceptance/bundle/run/basic/foo.py @@ -0,0 +1 @@ +print(1) diff --git a/acceptance/bundle/run/basic/output.txt b/acceptance/bundle/run/basic/output.txt index 92641a209d..f438a8a5e2 100644 --- a/acceptance/bundle/run/basic/output.txt +++ b/acceptance/bundle/run/basic/output.txt @@ -1,9 +1,11 @@ +=== no run key specified >>> [CLI] bundle run Error: expected a KEY of the resource to run Exit code: 1 +=== deploy and run resource >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/caterpillar/default/files... Deploying resources... @@ -13,4 +15,20 @@ Deployment complete! >>> [CLI] bundle run foo Run URL: [DATABRICKS_URL]/job/run/1 -2025-04-09 18:22:15 "foobar" TERMINATED +2025-04-09 18:30:30 "foobar" TERMINATED + +=== no resource key with -- +>>> [CLI] bundle run -- +Error: expected a KEY of the resource to run + +Exit code: 1 + +=== resource key with parameters +>>> [CLI] bundle run foo -- arg1 arg2 +Run URL: [DATABRICKS_URL]/job/run/1 + +2025-04-09 18:30:30 "foobar" TERMINATED + +=== inline script +>>> [CLI] bundle run -- echo hello +hello diff --git a/acceptance/bundle/run/basic/script b/acceptance/bundle/run/basic/script index cc2c3d32fc..d5d10c2a29 100644 --- a/acceptance/bundle/run/basic/script +++ b/acceptance/bundle/run/basic/script @@ -1,5 +1,16 @@ +title "no run key specified" errcode trace $CLI bundle run +title "deploy and run resource" errcode trace $CLI bundle deploy errcode trace $CLI bundle run foo + +title "no resource key with --" +errcode trace $CLI bundle run -- + +title "resource key with parameters" +errcode trace $CLI bundle run foo -- arg1 arg2 + +title "inline script" +errcode trace $CLI bundle run -- echo "hello" From 8056fd25d9265ffa9f289d06e7b76bfb2b33ce48 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 18:55:38 +0200 Subject: [PATCH 68/77] - --- acceptance/bundle/run/basic/output.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/run/basic/output.txt b/acceptance/bundle/run/basic/output.txt index f438a8a5e2..7cbf9b1027 100644 --- a/acceptance/bundle/run/basic/output.txt +++ b/acceptance/bundle/run/basic/output.txt @@ -15,7 +15,7 @@ Deployment complete! >>> [CLI] bundle run foo Run URL: [DATABRICKS_URL]/job/run/1 -2025-04-09 18:30:30 "foobar" TERMINATED +2025-04-09 18:30:30 "foobar" TERMINATED === no resource key with -- >>> [CLI] bundle run -- @@ -27,7 +27,7 @@ Exit code: 1 >>> [CLI] bundle run foo -- arg1 arg2 Run URL: [DATABRICKS_URL]/job/run/1 -2025-04-09 18:30:30 "foobar" TERMINATED +2025-04-09 18:30:30 "foobar" TERMINATED === inline script >>> [CLI] bundle run -- echo hello From a6f06940614bce5e4d42e0555a82e2e05c892b1c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 9 Apr 2025 19:02:40 +0200 Subject: [PATCH 69/77] - --- acceptance/bundle/run/basic/output.txt | 4 ++-- acceptance/bundle/run/basic/test.toml | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/run/basic/test.toml diff --git a/acceptance/bundle/run/basic/output.txt b/acceptance/bundle/run/basic/output.txt index 7cbf9b1027..9d73aa2985 100644 --- a/acceptance/bundle/run/basic/output.txt +++ b/acceptance/bundle/run/basic/output.txt @@ -15,7 +15,7 @@ Deployment complete! >>> [CLI] bundle run foo Run URL: [DATABRICKS_URL]/job/run/1 -2025-04-09 18:30:30 "foobar" TERMINATED +[DATE] HH:MM:SS "foobar" TERMINATED === no resource key with -- >>> [CLI] bundle run -- @@ -27,7 +27,7 @@ Exit code: 1 >>> [CLI] bundle run foo -- arg1 arg2 Run URL: [DATABRICKS_URL]/job/run/1 -2025-04-09 18:30:30 "foobar" TERMINATED +[DATE] HH:MM:SS "foobar" TERMINATED === inline script >>> [CLI] bundle run -- echo hello diff --git a/acceptance/bundle/run/basic/test.toml b/acceptance/bundle/run/basic/test.toml new file mode 100644 index 0000000000..fbd2506c39 --- /dev/null +++ b/acceptance/bundle/run/basic/test.toml @@ -0,0 +1,7 @@ +[[Repls]] +Old = "(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]" +New = "HH:MM:SS" + +[[Repls]] +Old = '20\d\d-\d\d-\d\d' +New = '[DATE]' From fd43bd70090c183d92322eeb09452d5e84723d4b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 9 May 2025 14:31:47 +0200 Subject: [PATCH 70/77] checkout from main --- cmd/bundle/generate/app.go | 60 +--------------------------------- cmd/bundle/run.go | 29 ++++++++--------- cmd/bundle/sync.go | 13 ++++---- cmd/root/auth.go | 10 ++++-- cmd/root/bundle.go | 6 ++-- cmd/root/bundle_test.go | 6 ++-- cmd/root/root.go | 16 +++++++++ libs/exec/execv.go | 19 ++++------- libs/exec/execv_unix.go | 3 +- libs/exec/execv_windows.go | 66 ++++++-------------------------------- 10 files changed, 70 insertions(+), 158 deletions(-) diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go index bcbbda7e87..427a01878f 100644 --- a/cmd/bundle/generate/app.go +++ b/cmd/bundle/generate/app.go @@ -1,11 +1,7 @@ package generate import ( - "context" - "errors" "fmt" - "io" - "io/fs" "path/filepath" "github.com/databricks/cli/bundle/config/generate" @@ -13,13 +9,9 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlsaver" - "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/textutil" - "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/spf13/cobra" - - "gopkg.in/yaml.v3" ) func NewGenerateAppCommand() *cobra.Command { @@ -62,18 +54,13 @@ func NewGenerateAppCommand() *cobra.Command { return err } - appConfig, err := getAppConfig(ctx, app, w) - if err != nil { - return fmt.Errorf("failed to get app config: %w", err) - } - // Making sure the source code path is relative to the config directory. rel, err := filepath.Rel(configDir, sourceDir) if err != nil { return err } - v, err := generate.ConvertAppToValue(app, filepath.ToSlash(rel), appConfig) + v, err := generate.ConvertAppToValue(app, filepath.ToSlash(rel)) if err != nil { return err } @@ -91,12 +78,6 @@ func NewGenerateAppCommand() *cobra.Command { }), } - // If there are app.yaml or app.yml files in the source code path, they will be downloaded but we don't want to include them in the bundle. - // We include this configuration inline, so we need to remove these files. - for _, configFile := range []string{"app.yml", "app.yaml"} { - delete(downloader.files, filepath.Join(sourceDir, configFile)) - } - err = downloader.FlushToDisk(ctx, force) if err != nil { return err @@ -116,42 +97,3 @@ func NewGenerateAppCommand() *cobra.Command { return cmd } - -func getAppConfig(ctx context.Context, app *apps.App, w *databricks.WorkspaceClient) (map[string]any, error) { - sourceCodePath := app.DefaultSourceCodePath - - f, err := filer.NewWorkspaceFilesClient(w, sourceCodePath) - if err != nil { - return nil, err - } - - // The app config is stored in app.yml or app.yaml file in the source code path. - configFileNames := []string{"app.yml", "app.yaml"} - for _, configFile := range configFileNames { - r, err := f.Read(ctx, configFile) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - continue - } - return nil, err - } - defer r.Close() - - cmdio.LogString(ctx, "Reading app configuration from "+configFile) - content, err := io.ReadAll(r) - if err != nil { - return nil, err - } - - var appConfig map[string]any - err = yaml.Unmarshal(content, &appConfig) - if err != nil { - cmdio.LogString(ctx, fmt.Sprintf("Failed to parse app configuration:\n%s\nerr: %v", string(content), err)) - return nil, nil - } - - return appConfig, nil - } - - return nil, nil -} diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 69e7a0f583..3713f0b80b 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -5,9 +5,11 @@ import ( "encoding/json" "errors" "fmt" + "os" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/resources" "github.com/databricks/cli/bundle/run" @@ -102,9 +104,6 @@ task or a Python wheel task, the second example applies. You can also use the bundle run command to execute scripts / commands in the same authentication context as the bundle. -Note: The current working directory of the provided command will be set to the root -of the bundle. - Authentication to the input command will be provided by setting the appropriate environment variables that Databricks tools use to authenticate. @@ -239,13 +238,13 @@ Example usage: } func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { - env := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) + cmdEnv := auth.ProcessEnv(cmdctx.ConfigUsed(cmd.Context())) // If user has specified a target, pass it to the child command. // // This is only useful for when the Databricks CLI is the child command. - if target := root.GetTarget(cmd); target != "" { - env = append(env, "DATABRICKS_CONFIG_TARGET="+target) + if b.Config.Bundle.Target != "" { + cmdEnv = append(cmdEnv, env.TargetVariable+"="+b.Config.Bundle.Target) } // If the bundle has a profile configured, explicitly pass it to the child command. @@ -254,19 +253,17 @@ func executeInline(cmd *cobra.Command, args []string, b *bundle.Bundle) error { // since if we do not explicitly pass the profile, the CLI will use the // auth configured in the bundle YAML configuration (if any). if b.Config.Workspace.Profile != "" { - env = append(env, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) + cmdEnv = append(cmdEnv, "DATABRICKS_CONFIG_PROFILE="+b.Config.Workspace.Profile) + } + + dir, err := os.Getwd() + if err != nil { + return err } return exec.Execv(exec.ExecvOptions{ Args: args, - Env: env, - // Execute all scripts from the bundle root directory. This behavior can - // be surprising in isolation, but we do it to keep the behavior consistent - // for both these cases: - // 1. One shot commands like `databricks bundle exec -- echo hello` - // 2. (upcoming) Scripts that are defined in the scripts section of the DAB. - Dir: b.BundleRootPath, - Stderr: cmd.ErrOrStderr(), - Stdout: cmd.OutOrStdout(), + Env: cmdEnv, + Dir: dir, }) } diff --git a/cmd/bundle/sync.go b/cmd/bundle/sync.go index 5674e676b2..65cda6d6c9 100644 --- a/cmd/bundle/sync.go +++ b/cmd/bundle/sync.go @@ -22,8 +22,7 @@ type syncFlags struct { full bool watch bool output flags.Output - exclude []string - include []string + dryRun bool } func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) (*sync.SyncOptions, error) { @@ -49,8 +48,7 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle) opts.Full = f.full opts.PollInterval = f.interval - opts.Exclude = append(opts.Exclude, f.exclude...) - opts.Include = append(opts.Include, f.include...) + opts.DryRun = f.dryRun return opts, nil } @@ -66,8 +64,7 @@ func newSyncCommand() *cobra.Command { cmd.Flags().BoolVar(&f.full, "full", false, "perform full synchronization (default is incremental)") cmd.Flags().BoolVar(&f.watch, "watch", false, "watch local file system for changes") cmd.Flags().Var(&f.output, "output", "type of the output format") - cmd.Flags().StringSliceVar(&f.exclude, "exclude", nil, "patterns to exclude from sync (can be specified multiple times)") - cmd.Flags().StringSliceVar(&f.include, "include", nil, "patterns to include in sync (can be specified multiple times)") + cmd.Flags().BoolVar(&f.dryRun, "dry-run", false, "simulate sync execution without making actual changes") cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -95,6 +92,10 @@ func newSyncCommand() *cobra.Command { log.Infof(ctx, "Remote file sync location: %v", opts.RemotePath) + if opts.DryRun { + log.Warnf(ctx, "Running in dry-run mode. No actual changes will be made.") + } + if f.watch { return s.RunContinuous(ctx) } diff --git a/cmd/root/auth.go b/cmd/root/auth.go index 15e32b9cd4..406631d24b 100644 --- a/cmd/root/auth.go +++ b/cmd/root/auth.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/databrickscfg/profile" @@ -137,7 +138,7 @@ func MustAccountClient(cmd *cobra.Command, args []string) error { allowPrompt := !hasProfileFlag && !shouldSkipPrompt(cmd.Context()) a, err := accountClientOrPrompt(cmd.Context(), cfg, allowPrompt) if err != nil { - return err + return renderError(ctx, cfg, err) } ctx = cmdctx.SetAccountClient(ctx, a) @@ -220,7 +221,7 @@ func MustWorkspaceClient(cmd *cobra.Command, args []string) error { allowPrompt := !hasProfileFlag && !shouldSkipPrompt(cmd.Context()) w, err := workspaceClientOrPrompt(cmd.Context(), cfg, allowPrompt) if err != nil { - return err + return renderError(ctx, cfg, err) } ctx = cmdctx.SetWorkspaceClient(ctx, w) @@ -306,3 +307,8 @@ func emptyHttpRequest(ctx context.Context) *http.Request { } return req } + +func renderError(ctx context.Context, cfg *config.Config, err error) error { + err, _ = auth.RewriteAuthError(ctx, cfg.Host, cfg.AccountID, cfg.Profile, err) + return err +} diff --git a/cmd/root/bundle.go b/cmd/root/bundle.go index a5fc24021a..99c278e2f5 100644 --- a/cmd/root/bundle.go +++ b/cmd/root/bundle.go @@ -13,8 +13,8 @@ import ( "golang.org/x/exp/maps" ) -// GetTarget returns the name of the target to operate in. -func GetTarget(cmd *cobra.Command) (value string) { +// getTarget returns the name of the target to operate in. +func getTarget(cmd *cobra.Command) (value string) { target, isFlagSet := targetFlagValue(cmd) if isFlagSet { return target @@ -78,7 +78,7 @@ func configureBundle(cmd *cobra.Command, b *bundle.Bundle) (*bundle.Bundle, diag // Load bundle and select target. ctx := cmd.Context() var diags diag.Diagnostics - if target := GetTarget(cmd); target == "" { + if target := getTarget(cmd); target == "" { diags = phases.LoadDefaultTarget(ctx, b) } else { diags = phases.LoadNamedTarget(ctx, b, target) diff --git a/cmd/root/bundle_test.go b/cmd/root/bundle_test.go index 3765b3e60b..45a476ff3f 100644 --- a/cmd/root/bundle_test.go +++ b/cmd/root/bundle_test.go @@ -216,7 +216,7 @@ func TestTargetFlagFull(t *testing.T) { err := Execute(ctx, cmd) assert.NoError(t, err) - assert.Equal(t, "development", GetTarget(cmd)) + assert.Equal(t, "development", getTarget(cmd)) } func TestTargetFlagShort(t *testing.T) { @@ -228,7 +228,7 @@ func TestTargetFlagShort(t *testing.T) { err := Execute(ctx, cmd) assert.NoError(t, err) - assert.Equal(t, "production", GetTarget(cmd)) + assert.Equal(t, "production", getTarget(cmd)) } // TODO: remove when environment flag is fully deprecated @@ -242,5 +242,5 @@ func TestTargetEnvironmentFlag(t *testing.T) { err := Execute(ctx, cmd) assert.NoError(t, err) - assert.Equal(t, "development", GetTarget(cmd)) + assert.Equal(t, "development", getTarget(cmd)) } diff --git a/cmd/root/root.go b/cmd/root/root.go index 8b1f976ea4..e18fea1329 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -185,3 +185,19 @@ Stack Trace: return err } + +// This function is used to report an unknown subcommand. +// It is used in the [cobra.Command.RunE] field of commands that have subcommands. +// If user provided a valid subcommand, RunE for the +// If there are any arguments, it means the user has provided an unknown subcommand. +// If there are no arguments, it means the user has not provided any subcommand, and the help +// command should be displayed. +func ReportUnknownSubcommand(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + return &InvalidArgsError{ + Message: fmt.Sprintf("unknown command %q for %q", args[0], cmd.CommandPath()), + Command: cmd, + } + } + return cmd.Help() +} diff --git a/libs/exec/execv.go b/libs/exec/execv.go index c477d76dd4..542de00574 100644 --- a/libs/exec/execv.go +++ b/libs/exec/execv.go @@ -1,20 +1,15 @@ package exec -import "io" - -// TODO(shreyas): Add a DATABRICKS_BUNDLE_INITIAL_CWD environment variable -// that users can read to figure out the original CWD. I'll do that when -// adding support for the scripts section. type ExecvOptions struct { + // Args is the name of the command to run and its arguments. + // Eg: ["echo", "hello"] for "echo hello" Args []string - Env []string - Dir string - // Stderr and Stdout are only used for Windows where execv is not supported. - // For Unix, the Stdout and Stderr are automatically inherited during the exec - // system call. - Stderr io.Writer - Stdout io.Writer + // Env is set the environment variables to set in the child process. + Env []string + + // Dir is the working directory of the child process. + Dir string } func Execv(opts ExecvOptions) error { diff --git a/libs/exec/execv_unix.go b/libs/exec/execv_unix.go index bbb896ad03..e81e51d07d 100644 --- a/libs/exec/execv_unix.go +++ b/libs/exec/execv_unix.go @@ -15,7 +15,8 @@ func execv(opts ExecvOptions) error { return fmt.Errorf("changing directory to %s failed: %w", opts.Dir, err) } - // execve syscall does not perform PATH lookup and + // execve syscall does not perform PATH lookup. Thus we need to query path + // before making the exec syscall. path, err := exec.LookPath(opts.Args[0]) if err != nil { return fmt.Errorf("looking up %q failed: %w", opts.Args[0], err) diff --git a/libs/exec/execv_windows.go b/libs/exec/execv_windows.go index 1fd525a8e4..4bb73973d1 100644 --- a/libs/exec/execv_windows.go +++ b/libs/exec/execv_windows.go @@ -3,17 +3,15 @@ package exec import ( - "bufio" "fmt" "os" "os/exec" "strings" - "sync" ) // Note: Windows does not support an execv syscall that replaces the current process. -// Thus we emulate that by launching a child process and streaming the output and returning -// the exit code. +// To emulate this, we create a child process, pass the stdin, stdout and stderr file descriptors, +// and return the exit code. // ref: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/execv-wexecv?view=msvc-170 func execv(opts ExecvOptions) error { path, err := exec.LookPath(opts.Args[0]) @@ -23,66 +21,18 @@ func execv(opts ExecvOptions) error { cmd := exec.Command(path, opts.Args[1:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Dir = opts.Dir cmd.Env = opts.Env - stdout, err := cmd.StdoutPipe() - if err != nil { - return fmt.Errorf("creating stdout pipe failed: %w", err) - } - - stderr, err := cmd.StderrPipe() - if err != nil { - return fmt.Errorf("creating stderr pipe failed: %w", err) - } - - // Start the child command. err = cmd.Start() if err != nil { return fmt.Errorf(" %s failed: %w", strings.Join(opts.Args, " "), err) } - var wg sync.WaitGroup - wg.Add(2) - - var stdoutErr error - go func() { - defer wg.Done() - - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - _, err = fmt.Fprintln(opts.Stdout, scanner.Text()) - if err != nil { - stdoutErr = err - break - } - } - }() - - var stderrErr error - go func() { - defer wg.Done() - - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - _, err = fmt.Fprintln(opts.Stderr, scanner.Text()) - if err != nil { - stderrErr = err - break - } - } - }() - - wg.Wait() - - if stdoutErr != nil { - return fmt.Errorf("writing stdout failed: %w", stdoutErr) - } - - if stderrErr != nil { - return fmt.Errorf("writing stderr failed: %w", stderrErr) - } - err = cmd.Wait() if exitErr, ok := err.(*exec.ExitError); ok { os.Exit(exitErr.ExitCode()) @@ -91,5 +41,9 @@ func execv(opts ExecvOptions) error { return fmt.Errorf("running %s failed: %w", strings.Join(opts.Args, " "), err) } + // Unix implementation of execv never returns control to the CLI process. + // To emulate this behavior, we exit early here if the child process exits + // successfully. + os.Exit(0) return nil } From ff71a220edb9afcefcf4177dbf1c3fe4f5de60dd Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 9 May 2025 14:34:02 +0200 Subject: [PATCH 71/77] - --- .../bundle/run/inline-script/basic/output.txt | 22 ++++++- .../target-is-passed/default/out.requests.txt | 39 +++++++++++++ .../target-is-passed/default/output.txt | 4 +- .../from_flag/out.requests.txt | 57 +++++++++++++++++++ .../target-is-passed/from_flag/output.txt | 4 +- 5 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt create mode 100644 acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt diff --git a/acceptance/bundle/run/inline-script/basic/output.txt b/acceptance/bundle/run/inline-script/basic/output.txt index a5b73dee9b..385f775966 100644 --- a/acceptance/bundle/run/inline-script/basic/output.txt +++ b/acceptance/bundle/run/inline-script/basic/output.txt @@ -1,3 +1,21 @@ -script: line 86: syntax error near unexpected token `<<<' -Exit code: 2 +>>> [CLI] bundle run -- echo hello, world +hello, world + +>>> [CLI] bundle run -- --help +Error: looking up "--help" failed: exec: "--help": executable file not found in PATH + +Exit code: 1 + +>>> [CLI] bundle run -- bash -c exit 5 + +Exit code: 5 + +>>> cat stderr.txt +Hello + +>>> cat - +abc + +>>> [CLI] bundle run -- printf hello +hello diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt new file mode 100644 index 0000000000..57a637ce0a --- /dev/null +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.requests.txt @@ -0,0 +1,39 @@ +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/workspace/get-status" +} +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/foobar/pat/files" + } +} +{ + "headers": { + "Authorization": [ + "Bearer [DATABRICKS_TOKEN]" + ] + }, + "method": "GET", + "path": "/api/2.0/workspace/get-status" +} diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt index 0f67bc39e9..513329a539 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/output.txt @@ -1,3 +1,3 @@ -script: line 85: syntax error near unexpected token `<<<' -Exit code: 2 +>>> [CLI] bundle run -- [CLI] bundle validate -o json +"pat" diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt new file mode 100644 index 0000000000..f4fb08cab2 --- /dev/null +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.requests.txt @@ -0,0 +1,57 @@ +{ + "method": "GET", + "path": "/oidc/.well-known/oauth-authorization-server" +} +{ + "method": "GET", + "path": "/oidc/.well-known/oauth-authorization-server" +} +{ + "headers": { + "Authorization": [ + "Basic [ENCODED_AUTH]" + ] + }, + "method": "POST", + "path": "/oidc/v1/token", + "raw_body": "grant_type=client_credentials\u0026scope=all-apis" +} +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "GET", + "path": "/api/2.0/workspace/get-status" +} +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/foobar/oauth/files" + } +} +{ + "headers": { + "Authorization": [ + "Bearer oauth-token" + ] + }, + "method": "GET", + "path": "/api/2.0/workspace/get-status" +} diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt index 0f67bc39e9..fb8a329289 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/output.txt @@ -1,3 +1,3 @@ -script: line 85: syntax error near unexpected token `<<<' -Exit code: 2 +>>> [CLI] bundle run -t oauth -- [CLI] bundle validate -o json +"oauth" From 576a16ed6b3dad3d21f8aad576d039cdff8fe787 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 9 May 2025 14:34:36 +0200 Subject: [PATCH 72/77] - --- .../profile-is-passed/from_bundle/.databrickscfg | 3 --- .../profile-is-passed/from_bundle/databricks.yml | 5 ----- .../profile-is-passed/from_bundle/out.requests.txt | 9 --------- .../profile-is-passed/from_bundle/output.txt | 6 ------ .../profile-is-passed/from_bundle/script | 10 ---------- 5 files changed, 33 deletions(-) delete mode 100644 acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg delete mode 100644 acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml delete mode 100644 acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt delete mode 100644 acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt delete mode 100644 acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg deleted file mode 100644 index 405e344a02..0000000000 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/.databrickscfg +++ /dev/null @@ -1,3 +0,0 @@ -[someprofile] -host = $DATABRICKS_HOST -token = $DATABRICKS_TOKEN diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml deleted file mode 100644 index 5de7d1d968..0000000000 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/databricks.yml +++ /dev/null @@ -1,5 +0,0 @@ -bundle: - name: foobar - -workspace: - profile: someprofile diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt deleted file mode 100644 index c4a1034826..0000000000 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/out.requests.txt +++ /dev/null @@ -1,9 +0,0 @@ -{ - "headers": { - "Authorization": [ - "Bearer [DATABRICKS_TOKEN]" - ] - }, - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt deleted file mode 100644 index 704a6ac915..0000000000 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/output.txt +++ /dev/null @@ -1,6 +0,0 @@ - ->>> [CLI] bundle run -- [CLI] current-user me -{ - "id":"[USERID]", - "userName":"[USERNAME]" -} diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script deleted file mode 100644 index 28bc0f1b84..0000000000 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_bundle/script +++ /dev/null @@ -1,10 +0,0 @@ -# Replace placeholder with an actual host URL and token -envsubst < .databrickscfg > out && mv out .databrickscfg -export DATABRICKS_CONFIG_FILE=.databrickscfg - -# Credentials will be picked up from .databrickscfg. Unset existing credentials. -unset DATABRICKS_HOST -unset DATABRICKS_TOKEN - -# This should use the profile configured in the bundle, i.e. PAT auth -trace $CLI bundle run -- $CLI current-user me From 1f36d7421a779ef4cf237488ae123633f81dfa24 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 9 May 2025 14:44:15 +0200 Subject: [PATCH 73/77] undo fix --- bundle/run/job.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/run/job.go b/bundle/run/job.go index dd62fd324e..dbd2ffcc78 100644 --- a/bundle/run/job.go +++ b/bundle/run/job.go @@ -188,7 +188,7 @@ func (r *jobRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, e waiter, err := w.Jobs.RunNow(ctx, *req) if err != nil { - return nil, fmt.Errorf("cannot start job: %w", err) + return nil, errors.New("cannot start job") } if opts.NoWait { From 4c327f56c6e7b36c1f9d28200aee778ba094c01f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 9 May 2025 14:55:45 +0200 Subject: [PATCH 74/77] add stateful tracking for runs --- acceptance/bundle/run/basic/output.txt | 8 +++--- acceptance/internal/handlers.go | 10 +------ libs/testserver/fake_workspace.go | 40 ++++++++++++++++++++++++-- libs/testserver/server.go | 2 +- 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/acceptance/bundle/run/basic/output.txt b/acceptance/bundle/run/basic/output.txt index 9d73aa2985..992dfcc1dc 100644 --- a/acceptance/bundle/run/basic/output.txt +++ b/acceptance/bundle/run/basic/output.txt @@ -13,9 +13,9 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle run foo -Run URL: [DATABRICKS_URL]/job/run/1 +Error: cannot start job -[DATE] HH:MM:SS "foobar" TERMINATED +Exit code: 1 === no resource key with -- >>> [CLI] bundle run -- @@ -25,9 +25,9 @@ Exit code: 1 === resource key with parameters >>> [CLI] bundle run foo -- arg1 arg2 -Run URL: [DATABRICKS_URL]/job/run/1 +Error: cannot start job -[DATE] HH:MM:SS "foobar" TERMINATED +Exit code: 1 === inline script >>> [CLI] bundle run -- echo hello diff --git a/acceptance/internal/handlers.go b/acceptance/internal/handlers.go index 3a26f94df0..ee897ae2bd 100644 --- a/acceptance/internal/handlers.go +++ b/acceptance/internal/handlers.go @@ -217,15 +217,7 @@ func addDefaultHandlers(server *testserver.Server) { } } - return jobs.Run{ - RunId: runIdInt, - State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - }, - RunPageUrl: fmt.Sprintf("%s/job/run/%d", server.URL, runIdInt), - RunType: jobs.RunTypeJobRun, - RunName: "foobar", - } + return req.Workspace.JobsGetRun(runIdInt) }) server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(_ testserver.Request) any { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 1e08b1b508..4afdb77f04 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -22,11 +22,16 @@ import ( type FakeWorkspace struct { mu sync.Mutex + // URL of the workspace server. + url string + directories map[string]bool files map[string][]byte // normally, ids are not sequential, but we make them sequential for deterministic diff nextJobId int64 jobs map[int64]jobs.Job + jobRuns map[int64]jobs.Run + runCount int64 Pipelines map[string]pipelines.PipelineSpec Monitors map[string]catalog.MonitorInfo @@ -70,8 +75,9 @@ func MapDelete[T any](w *FakeWorkspace, collection map[string]T, key string) Res return Response{} } -func NewFakeWorkspace() *FakeWorkspace { +func NewFakeWorkspace(url string) *FakeWorkspace { return &FakeWorkspace{ + url: url, directories: map[string]bool{ "/Workspace": true, }, @@ -248,6 +254,7 @@ func (s *FakeWorkspace) JobsGet(jobId string) Response { } func (s *FakeWorkspace) JobsRunNow(jobId int64) Response { + defer s.LockUnlock()() _, ok := s.jobs[jobId] if !ok { return Response{ @@ -255,13 +262,42 @@ func (s *FakeWorkspace) JobsRunNow(jobId int64) Response { } } + runId := s.runCount + s.jobRuns[runId] = jobs.Run{ + RunId: runId, + State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateRunning, + }, + RunPageUrl: fmt.Sprintf("%s/job/run/%d", s.url, runId), + RunType: jobs.RunTypeJobRun, + RunName: "run-name", + } + s.runCount++ + return Response{ Body: jobs.RunNowResponse{ - RunId: 1, + RunId: runId, }, } } +func (s *FakeWorkspace) JobsGetRun(runId int64) Response { + defer s.LockUnlock()() + + run, ok := s.jobRuns[runId] + if !ok { + return Response{ + StatusCode: 404, + } + } + + // Terminate the run on the first GET call. + run.State.LifeCycleState = jobs.RunLifeCycleStateTerminated + return Response{ + Body: run, + } +} + func (s *FakeWorkspace) PipelinesGet(pipelineId string) Response { defer s.LockUnlock()() diff --git a/libs/testserver/server.go b/libs/testserver/server.go index f118e59911..a9866843e1 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -240,7 +240,7 @@ func (s *Server) getWorkspaceForToken(token string) *FakeWorkspace { defer s.mu.Unlock() if _, ok := s.fakeWorkspaces[token]; !ok { - s.fakeWorkspaces[token] = NewFakeWorkspace() + s.fakeWorkspaces[token] = NewFakeWorkspace(s.Server.URL) } return s.fakeWorkspaces[token] From 2a06b498ab059fc579a7b7d4fbe5fc64dd1bf4c2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 9 May 2025 15:31:08 +0200 Subject: [PATCH 75/77] Revert "add stateful tracking for runs" This reverts commit 4c327f56c6e7b36c1f9d28200aee778ba094c01f. --- acceptance/bundle/run/basic/output.txt | 8 +++--- acceptance/internal/handlers.go | 10 ++++++- libs/testserver/fake_workspace.go | 40 ++------------------------ libs/testserver/server.go | 2 +- 4 files changed, 16 insertions(+), 44 deletions(-) diff --git a/acceptance/bundle/run/basic/output.txt b/acceptance/bundle/run/basic/output.txt index 992dfcc1dc..9d73aa2985 100644 --- a/acceptance/bundle/run/basic/output.txt +++ b/acceptance/bundle/run/basic/output.txt @@ -13,9 +13,9 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle run foo -Error: cannot start job +Run URL: [DATABRICKS_URL]/job/run/1 -Exit code: 1 +[DATE] HH:MM:SS "foobar" TERMINATED === no resource key with -- >>> [CLI] bundle run -- @@ -25,9 +25,9 @@ Exit code: 1 === resource key with parameters >>> [CLI] bundle run foo -- arg1 arg2 -Error: cannot start job +Run URL: [DATABRICKS_URL]/job/run/1 -Exit code: 1 +[DATE] HH:MM:SS "foobar" TERMINATED === inline script >>> [CLI] bundle run -- echo hello diff --git a/acceptance/internal/handlers.go b/acceptance/internal/handlers.go index ee897ae2bd..3a26f94df0 100644 --- a/acceptance/internal/handlers.go +++ b/acceptance/internal/handlers.go @@ -217,7 +217,15 @@ func addDefaultHandlers(server *testserver.Server) { } } - return req.Workspace.JobsGetRun(runIdInt) + return jobs.Run{ + RunId: runIdInt, + State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + }, + RunPageUrl: fmt.Sprintf("%s/job/run/%d", server.URL, runIdInt), + RunType: jobs.RunTypeJobRun, + RunName: "foobar", + } }) server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(_ testserver.Request) any { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 4afdb77f04..1e08b1b508 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -22,16 +22,11 @@ import ( type FakeWorkspace struct { mu sync.Mutex - // URL of the workspace server. - url string - directories map[string]bool files map[string][]byte // normally, ids are not sequential, but we make them sequential for deterministic diff nextJobId int64 jobs map[int64]jobs.Job - jobRuns map[int64]jobs.Run - runCount int64 Pipelines map[string]pipelines.PipelineSpec Monitors map[string]catalog.MonitorInfo @@ -75,9 +70,8 @@ func MapDelete[T any](w *FakeWorkspace, collection map[string]T, key string) Res return Response{} } -func NewFakeWorkspace(url string) *FakeWorkspace { +func NewFakeWorkspace() *FakeWorkspace { return &FakeWorkspace{ - url: url, directories: map[string]bool{ "/Workspace": true, }, @@ -254,7 +248,6 @@ func (s *FakeWorkspace) JobsGet(jobId string) Response { } func (s *FakeWorkspace) JobsRunNow(jobId int64) Response { - defer s.LockUnlock()() _, ok := s.jobs[jobId] if !ok { return Response{ @@ -262,42 +255,13 @@ func (s *FakeWorkspace) JobsRunNow(jobId int64) Response { } } - runId := s.runCount - s.jobRuns[runId] = jobs.Run{ - RunId: runId, - State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateRunning, - }, - RunPageUrl: fmt.Sprintf("%s/job/run/%d", s.url, runId), - RunType: jobs.RunTypeJobRun, - RunName: "run-name", - } - s.runCount++ - return Response{ Body: jobs.RunNowResponse{ - RunId: runId, + RunId: 1, }, } } -func (s *FakeWorkspace) JobsGetRun(runId int64) Response { - defer s.LockUnlock()() - - run, ok := s.jobRuns[runId] - if !ok { - return Response{ - StatusCode: 404, - } - } - - // Terminate the run on the first GET call. - run.State.LifeCycleState = jobs.RunLifeCycleStateTerminated - return Response{ - Body: run, - } -} - func (s *FakeWorkspace) PipelinesGet(pipelineId string) Response { defer s.LockUnlock()() diff --git a/libs/testserver/server.go b/libs/testserver/server.go index a9866843e1..f118e59911 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -240,7 +240,7 @@ func (s *Server) getWorkspaceForToken(token string) *FakeWorkspace { defer s.mu.Unlock() if _, ok := s.fakeWorkspaces[token]; !ok { - s.fakeWorkspaces[token] = NewFakeWorkspace(s.Server.URL) + s.fakeWorkspaces[token] = NewFakeWorkspace() } return s.fakeWorkspaces[token] From 72305f1b0d7bc9733aa1a202edfe79c479c8f92f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 9 May 2025 15:38:42 +0200 Subject: [PATCH 76/77] proper stateful tracking --- acceptance/bundle/run/basic/output.txt | 6 +-- acceptance/internal/handlers.go | 10 +---- libs/testserver/fake_workspace.go | 62 ++++++++++++++++++++------ libs/testserver/server.go | 2 +- 4 files changed, 54 insertions(+), 26 deletions(-) diff --git a/acceptance/bundle/run/basic/output.txt b/acceptance/bundle/run/basic/output.txt index 9d73aa2985..7366ed6727 100644 --- a/acceptance/bundle/run/basic/output.txt +++ b/acceptance/bundle/run/basic/output.txt @@ -15,7 +15,7 @@ Deployment complete! >>> [CLI] bundle run foo Run URL: [DATABRICKS_URL]/job/run/1 -[DATE] HH:MM:SS "foobar" TERMINATED +[DATE] HH:MM:SS "run-name" TERMINATED === no resource key with -- >>> [CLI] bundle run -- @@ -25,9 +25,9 @@ Exit code: 1 === resource key with parameters >>> [CLI] bundle run foo -- arg1 arg2 -Run URL: [DATABRICKS_URL]/job/run/1 +Run URL: [DATABRICKS_URL]/job/run/2 -[DATE] HH:MM:SS "foobar" TERMINATED +[DATE] HH:MM:SS "run-name" TERMINATED === inline script >>> [CLI] bundle run -- echo hello diff --git a/acceptance/internal/handlers.go b/acceptance/internal/handlers.go index 3a26f94df0..ee897ae2bd 100644 --- a/acceptance/internal/handlers.go +++ b/acceptance/internal/handlers.go @@ -217,15 +217,7 @@ func addDefaultHandlers(server *testserver.Server) { } } - return jobs.Run{ - RunId: runIdInt, - State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - }, - RunPageUrl: fmt.Sprintf("%s/job/run/%d", server.URL, runIdInt), - RunType: jobs.RunTypeJobRun, - RunName: "foobar", - } + return req.Workspace.JobsGetRun(runIdInt) }) server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(_ testserver.Request) any { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 1e08b1b508..de8e2b57e0 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -20,13 +20,16 @@ import ( // FakeWorkspace holds a state of a workspace for acceptance tests. type FakeWorkspace struct { - mu sync.Mutex + mu sync.Mutex + url string directories map[string]bool files map[string][]byte // normally, ids are not sequential, but we make them sequential for deterministic diff - nextJobId int64 - jobs map[int64]jobs.Job + nextJobId int64 + nextJobRunId int64 + jobs map[int64]jobs.Job + jobRuns map[int64]jobs.Run Pipelines map[string]pipelines.PipelineSpec Monitors map[string]catalog.MonitorInfo @@ -70,19 +73,21 @@ func MapDelete[T any](w *FakeWorkspace, collection map[string]T, key string) Res return Response{} } -func NewFakeWorkspace() *FakeWorkspace { +func NewFakeWorkspace(url string) *FakeWorkspace { return &FakeWorkspace{ + url: url, directories: map[string]bool{ "/Workspace": true, }, - files: map[string][]byte{}, - jobs: map[int64]jobs.Job{}, - nextJobId: 1, - - Pipelines: map[string]pipelines.PipelineSpec{}, - Monitors: map[string]catalog.MonitorInfo{}, - Apps: map[string]apps.App{}, - Schemas: map[string]catalog.SchemaInfo{}, + files: map[string][]byte{}, + jobs: map[int64]jobs.Job{}, + jobRuns: map[int64]jobs.Run{}, + nextJobId: 1, + nextJobRunId: 1, + Pipelines: map[string]pipelines.PipelineSpec{}, + Monitors: map[string]catalog.MonitorInfo{}, + Apps: map[string]apps.App{}, + Schemas: map[string]catalog.SchemaInfo{}, } } @@ -248,6 +253,8 @@ func (s *FakeWorkspace) JobsGet(jobId string) Response { } func (s *FakeWorkspace) JobsRunNow(jobId int64) Response { + defer s.LockUnlock()() + _, ok := s.jobs[jobId] if !ok { return Response{ @@ -255,13 +262,42 @@ func (s *FakeWorkspace) JobsRunNow(jobId int64) Response { } } + runId := s.nextJobRunId + s.nextJobRunId++ + s.jobRuns[runId] = jobs.Run{ + RunId: runId, + State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateRunning, + }, + RunPageUrl: fmt.Sprintf("%s/job/run/%d", s.url, runId), + RunType: jobs.RunTypeJobRun, + RunName: "run-name", + } + return Response{ Body: jobs.RunNowResponse{ - RunId: 1, + RunId: runId, }, } } +func (s *FakeWorkspace) JobsGetRun(runId int64) Response { + defer s.LockUnlock()() + + run, ok := s.jobRuns[runId] + if !ok { + return Response{ + StatusCode: 404, + } + } + + // Mark the run as terminated. + run.State.LifeCycleState = jobs.RunLifeCycleStateTerminated + return Response{ + Body: run, + } +} + func (s *FakeWorkspace) PipelinesGet(pipelineId string) Response { defer s.LockUnlock()() diff --git a/libs/testserver/server.go b/libs/testserver/server.go index f118e59911..a9866843e1 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -240,7 +240,7 @@ func (s *Server) getWorkspaceForToken(token string) *FakeWorkspace { defer s.mu.Unlock() if _, ok := s.fakeWorkspaces[token]; !ok { - s.fakeWorkspaces[token] = NewFakeWorkspace() + s.fakeWorkspaces[token] = NewFakeWorkspace(s.Server.URL) } return s.fakeWorkspaces[token] From 92bf4565b28fd8554527c75b90d005912deb792a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 9 May 2025 15:41:27 +0200 Subject: [PATCH 77/77] - --- .wsignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.wsignore b/.wsignore index d4831cd94e..d964e1577a 100644 --- a/.wsignore +++ b/.wsignore @@ -19,6 +19,7 @@ acceptance/selftest/record_cloud/volume-io/hello.txt # "bundle run" has trailing whitespace: acceptance/bundle/integration_whl/*/output.txt +acceptance/bundle/run/basic/output.txt # "bundle init" has trailing whitespace: acceptance/bundle/templates-machinery/helpers-error/output.txt