diff --git a/acceptance/pipelines/deploy/auto-approve/databricks.yml b/acceptance/pipelines/deploy/auto-approve/databricks.yml new file mode 100644 index 0000000000..5a93a7a11d --- /dev/null +++ b/acceptance/pipelines/deploy/auto-approve/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: test-pipeline-auto-approve + +include: + - "./*.yml" diff --git a/acceptance/pipelines/deploy/auto-approve/out.test.toml b/acceptance/pipelines/deploy/auto-approve/out.test.toml new file mode 100644 index 0000000000..387f29e8bb --- /dev/null +++ b/acceptance/pipelines/deploy/auto-approve/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform"] diff --git a/acceptance/pipelines/deploy/auto-approve/output.txt b/acceptance/pipelines/deploy/auto-approve/output.txt new file mode 100644 index 0000000000..fb1d852932 --- /dev/null +++ b/acceptance/pipelines/deploy/auto-approve/output.txt @@ -0,0 +1,36 @@ + +>>> [PIPELINES] deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto-approve/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Remove resources from configuration to test auto-approve +>>> rm resources.yml + +=== Try to redeploy without --auto-approve - should fail +>>> errcode [PIPELINES] deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto-approve/default/files... + +This action will result in the deletion or recreation of the following DLT Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed: + delete pipeline foo +Error: the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed + + +Exit code: 1 + +=== Redeploy with --auto-approve - should succeed +>>> [PIPELINES] deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto-approve/default/files... + +This action will result in the deletion or recreation of the following DLT Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed: + delete pipeline foo +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/pipelines/deploy/auto-approve/resources.yml b/acceptance/pipelines/deploy/auto-approve/resources.yml new file mode 100644 index 0000000000..5de4a7af20 --- /dev/null +++ b/acceptance/pipelines/deploy/auto-approve/resources.yml @@ -0,0 +1,4 @@ +resources: + pipelines: + foo: + name: test-pipeline diff --git a/acceptance/pipelines/deploy/auto-approve/script b/acceptance/pipelines/deploy/auto-approve/script new file mode 100644 index 0000000000..579031b617 --- /dev/null +++ b/acceptance/pipelines/deploy/auto-approve/script @@ -0,0 +1,10 @@ +trace $PIPELINES deploy + +title "Remove resources from configuration to test auto-approve" +trace rm resources.yml + +title "Try to redeploy without --auto-approve - should fail" +trace errcode $PIPELINES deploy + +title "Redeploy with --auto-approve - should succeed" +trace $PIPELINES deploy --auto-approve diff --git a/acceptance/pipelines/deploy/auto-approve/test.toml b/acceptance/pipelines/deploy/auto-approve/test.toml new file mode 100644 index 0000000000..892bca146b --- /dev/null +++ b/acceptance/pipelines/deploy/auto-approve/test.toml @@ -0,0 +1 @@ +EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] diff --git a/acceptance/pipelines/deploy/create-pipeline/databricks.yml b/acceptance/pipelines/deploy/create-pipeline/databricks.yml new file mode 100644 index 0000000000..90abf6c3f3 --- /dev/null +++ b/acceptance/pipelines/deploy/create-pipeline/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-create-pipeline + +resources: + pipelines: + foo: + name: test-pipeline + libraries: + - notebook: + path: ./nb.sql diff --git a/acceptance/pipelines/deploy/create-pipeline/nb.sql b/acceptance/pipelines/deploy/create-pipeline/nb.sql new file mode 100644 index 0000000000..199ff50788 --- /dev/null +++ b/acceptance/pipelines/deploy/create-pipeline/nb.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +select 1 diff --git a/acceptance/pipelines/deploy/create-pipeline/out.test.toml b/acceptance/pipelines/deploy/create-pipeline/out.test.toml new file mode 100644 index 0000000000..387f29e8bb --- /dev/null +++ b/acceptance/pipelines/deploy/create-pipeline/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform"] diff --git a/acceptance/pipelines/deploy/create-pipeline/output.txt b/acceptance/pipelines/deploy/create-pipeline/output.txt new file mode 100644 index 0000000000..6f37a52570 --- /dev/null +++ b/acceptance/pipelines/deploy/create-pipeline/output.txt @@ -0,0 +1,28 @@ + +>>> [PIPELINES] deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-create-pipeline/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] pipelines get [UUID] +{ + "spec": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-create-pipeline/default/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[UUID]", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/test-create-pipeline/default/files/nb" + } + } + ], + "name": "test-pipeline", + "storage": "dbfs:/pipelines/[UUID]" + } +} diff --git a/acceptance/pipelines/deploy/create-pipeline/script b/acceptance/pipelines/deploy/create-pipeline/script new file mode 100644 index 0000000000..3042dea425 --- /dev/null +++ b/acceptance/pipelines/deploy/create-pipeline/script @@ -0,0 +1,3 @@ +trace $PIPELINES deploy +PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | jq "{spec}" diff --git a/acceptance/pipelines/deploy/create-pipeline/test.toml b/acceptance/pipelines/deploy/create-pipeline/test.toml new file mode 100644 index 0000000000..892bca146b --- /dev/null +++ b/acceptance/pipelines/deploy/create-pipeline/test.toml @@ -0,0 +1 @@ +EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] diff --git a/acceptance/pipelines/deploy/render-diagnostics-warning/databricks.yml b/acceptance/pipelines/deploy/render-diagnostics-warning/databricks.yml new file mode 100644 index 0000000000..0129e0df44 --- /dev/null +++ b/acceptance/pipelines/deploy/render-diagnostics-warning/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: render-diagnostics-warning-pipeline + +resources: + pipelines: + test-pipeline: + # This is an unknown property that should trigger a warning + unknown_property: "this_should_trigger_a_warning" diff --git a/acceptance/pipelines/deploy/render-diagnostics-warning/out.test.toml b/acceptance/pipelines/deploy/render-diagnostics-warning/out.test.toml new file mode 100644 index 0000000000..387f29e8bb --- /dev/null +++ b/acceptance/pipelines/deploy/render-diagnostics-warning/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform"] diff --git a/acceptance/pipelines/deploy/render-diagnostics-warning/output.txt b/acceptance/pipelines/deploy/render-diagnostics-warning/output.txt new file mode 100644 index 0000000000..67030fbebc --- /dev/null +++ b/acceptance/pipelines/deploy/render-diagnostics-warning/output.txt @@ -0,0 +1,11 @@ + +>>> [PIPELINES] deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/render-diagnostics-warning-pipeline/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +Warning: unknown field: unknown_property + at resources.pipelines.test-pipeline + in databricks.yml:8:7 + + diff --git a/acceptance/pipelines/deploy/render-diagnostics-warning/script b/acceptance/pipelines/deploy/render-diagnostics-warning/script new file mode 100644 index 0000000000..735145a6c9 --- /dev/null +++ b/acceptance/pipelines/deploy/render-diagnostics-warning/script @@ -0,0 +1,3 @@ +trace $PIPELINES deploy +# print newline to comply with whitespace linter +printf "\n" diff --git a/acceptance/pipelines/deploy/render-diagnostics-warning/test.toml b/acceptance/pipelines/deploy/render-diagnostics-warning/test.toml new file mode 100644 index 0000000000..892bca146b --- /dev/null +++ b/acceptance/pipelines/deploy/render-diagnostics-warning/test.toml @@ -0,0 +1 @@ +EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] diff --git a/acceptance/pipelines/deploy/var-flag/databricks.yml b/acceptance/pipelines/deploy/var-flag/databricks.yml new file mode 100644 index 0000000000..5d116ce85a --- /dev/null +++ b/acceptance/pipelines/deploy/var-flag/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: test-pipeline-var-flag + +variables: + catalog: + default: main + schema: + default: test_schema + +resources: + pipelines: + foo: + catalog: ${var.catalog} + target: ${var.schema} diff --git a/acceptance/pipelines/deploy/var-flag/out.test.toml b/acceptance/pipelines/deploy/var-flag/out.test.toml new file mode 100644 index 0000000000..387f29e8bb --- /dev/null +++ b/acceptance/pipelines/deploy/var-flag/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform"] diff --git a/acceptance/pipelines/deploy/var-flag/output.txt b/acceptance/pipelines/deploy/var-flag/output.txt new file mode 100644 index 0000000000..a3d5510c75 --- /dev/null +++ b/acceptance/pipelines/deploy/var-flag/output.txt @@ -0,0 +1,14 @@ + +=== Test --var flag usage +>>> [PIPELINES] deploy --var=catalog=custom_catalog --var=schema=custom_schema --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-var-flag/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify: Check that variables were substituted correctly +>>> [CLI] pipelines get [UUID] +{ + "catalog": "custom_catalog", + "target": "custom_schema" +} diff --git a/acceptance/pipelines/deploy/var-flag/script b/acceptance/pipelines/deploy/var-flag/script new file mode 100644 index 0000000000..13e016eaed --- /dev/null +++ b/acceptance/pipelines/deploy/var-flag/script @@ -0,0 +1,7 @@ +title "Test --var flag usage" +trace $PIPELINES deploy --var="catalog=custom_catalog" --var="schema=custom_schema" --auto-approve + +title "Verify: Check that variables were substituted correctly" +PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | jq '.spec | {catalog: .catalog, target: .target}' +# Expected: catalog should be "custom_catalog" and target should contain "custom_schema" diff --git a/acceptance/pipelines/deploy/var-flag/test.toml b/acceptance/pipelines/deploy/var-flag/test.toml new file mode 100644 index 0000000000..892bca146b --- /dev/null +++ b/acceptance/pipelines/deploy/var-flag/test.toml @@ -0,0 +1 @@ +EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] diff --git a/acceptance/pipelines/install-pipelines-cli/output.txt b/acceptance/pipelines/install-pipelines-cli/output.txt index fe1df4f5c3..96bb84e37f 100644 --- a/acceptance/pipelines/install-pipelines-cli/output.txt +++ b/acceptance/pipelines/install-pipelines-cli/output.txt @@ -1,9 +1,9 @@ === install pipelines cli ->>> errcode [CLI] install-pipelines-cli -d ./subdir +>>> [CLI] install-pipelines-cli -d ./subdir pipelines successfully installed in directory "./subdir" ->>> errcode ./subdir/pipelines +>>> ./subdir/pipelines Pipelines CLI Usage: @@ -12,6 +12,7 @@ Usage: Available Commands: auth Authentication related commands completion Generate the autocompletion script for the specified shell + deploy Deploy pipelines help Help about any command init Initialize a new pipelines project @@ -21,12 +22,13 @@ Flags: -o, --output type output type: text or json (default text) -p, --profile string ~/.databrickscfg profile -t, --target string project target to use (if applicable) + --var strings set values for variables defined in project config. Example: --var="foo=bar" -v, --version version for pipelines Use "pipelines [command] --help" for more information about a command. === pipelines already installed ->>> errcode [CLI] install-pipelines-cli -d ./subdir +>>> [CLI] install-pipelines-cli -d ./subdir pipelines already installed in directory "./subdir" === pipelines file exists, should not overwrite @@ -36,10 +38,10 @@ Error: cannot install pipelines CLI: "subdir/pipelines" already exists Exit code: 1 === databricks executable called with alias ->>> errcode ./subdir/notdatabricks install-pipelines-cli -d ./subdir +>>> ./subdir/notdatabricks install-pipelines-cli -d ./subdir pipelines successfully installed in directory "./subdir" ->>> errcode ./subdir/pipelines +>>> ./subdir/pipelines Pipelines CLI Usage: @@ -48,6 +50,7 @@ Usage: Available Commands: auth Authentication related commands completion Generate the autocompletion script for the specified shell + deploy Deploy pipelines help Help about any command init Initialize a new pipelines project @@ -57,6 +60,7 @@ Flags: -o, --output type output type: text or json (default text) -p, --profile string ~/.databrickscfg profile -t, --target string project target to use (if applicable) + --var strings set values for variables defined in project config. Example: --var="foo=bar" -v, --version version for pipelines Use "pipelines [command] --help" for more information about a command. diff --git a/acceptance/pipelines/install-pipelines-cli/script b/acceptance/pipelines/install-pipelines-cli/script index 81eeb39764..2a4810fce8 100644 --- a/acceptance/pipelines/install-pipelines-cli/script +++ b/acceptance/pipelines/install-pipelines-cli/script @@ -3,11 +3,11 @@ pipelines="$tmpdir/pipelines" mkdir -p $tmpdir title "install pipelines cli" -trace errcode $CLI install-pipelines-cli -d $tmpdir -trace errcode $pipelines +trace $CLI install-pipelines-cli -d $tmpdir +trace $pipelines title "pipelines already installed" -trace errcode $CLI install-pipelines-cli -d $tmpdir +trace $CLI install-pipelines-cli -d $tmpdir rm -f $pipelines title "pipelines file exists, should not overwrite" @@ -17,9 +17,7 @@ rm -f $pipelines title "databricks executable called with alias" cp $CLI $tmpdir/notdatabricks -trace errcode $tmpdir/notdatabricks install-pipelines-cli -d $tmpdir -trace errcode $pipelines +trace $tmpdir/notdatabricks install-pipelines-cli -d $tmpdir +trace $pipelines -# Cleanup -rm -f $tmpdir/notdatabricks $pipelines -rm -rf $tmpdir +rm -rf $tmpdir/notdatabricks $pipelines $tmpdir diff --git a/acceptance/pipelines/test.toml b/acceptance/pipelines/test.toml new file mode 100644 index 0000000000..b2881a9045 --- /dev/null +++ b/acceptance/pipelines/test.toml @@ -0,0 +1,3 @@ +# All pipelines tests are local only +Local = true +Cloud = false diff --git a/cmd/pipelines/deploy.go b/cmd/pipelines/deploy.go new file mode 100644 index 0000000000..6594f2958e --- /dev/null +++ b/cmd/pipelines/deploy.go @@ -0,0 +1,69 @@ +// Copied from cmd/bundle/deploy.go and adapted for pipelines use. +package pipelines + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/sync" + "github.com/spf13/cobra" +) + +func deployCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "deploy", + Short: "Deploy pipelines", + Args: root.NoArgs, + } + + var forceLock bool + var autoApprove bool + var verbose bool + cmd.Flags().BoolVar(&forceLock, "force-lock", false, "Force acquisition of deployment lock.") + cmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip interactive approvals that might be required for deployment.") + cmd.Flags().BoolVar(&verbose, "verbose", false, "Enable verbose output.") + // Verbose flag currently only affects file sync output, it's used by the vscode extension + cmd.Flags().MarkHidden("verbose") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + b, diags := utils.ConfigureBundleWithVariables(cmd) + + if !diags.HasError() { + bundle.ApplyFunc(ctx, b, func(context.Context, *bundle.Bundle) diag.Diagnostics { + b.Config.Bundle.Deployment.Lock.Force = forceLock + b.AutoApprove = autoApprove + return nil + }) + + var outputHandler sync.OutputHandler + if verbose { + outputHandler = func(ctx context.Context, c <-chan sync.Event) { + sync.TextOutput(ctx, c, cmd.OutOrStdout()) + } + } + + diags = diags.Extend(phases.Initialize(ctx, b)) + + if !diags.HasError() { + diags = diags.Extend(bundle.Apply(ctx, b, validate.FastValidate())) + } + + if !diags.HasError() { + diags = diags.Extend(phases.Build(ctx, b)) + } + + if !diags.HasError() { + diags = diags.Extend(phases.Deploy(ctx, b, outputHandler)) + } + } + + return renderDiagnostics(cmd.OutOrStdout(), b, diags) + } + return cmd +} diff --git a/cmd/pipelines/pipelines.go b/cmd/pipelines/pipelines.go index 61808cc60e..a7838ebdd2 100644 --- a/cmd/pipelines/pipelines.go +++ b/cmd/pipelines/pipelines.go @@ -9,7 +9,9 @@ import ( func New(ctx context.Context) *cobra.Command { cli := root.New(ctx) + initVariableFlag(cli) cli.AddCommand(initCommand()) + cli.AddCommand(deployCommand()) cli.AddCommand(authCommand()) return cli } diff --git a/cmd/pipelines/utils.go b/cmd/pipelines/utils.go new file mode 100644 index 0000000000..66faf65ae7 --- /dev/null +++ b/cmd/pipelines/utils.go @@ -0,0 +1,27 @@ +package pipelines + +import ( + "fmt" + "io" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/render" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/diag" +) + +// renderDiagnostics renders the diagnostics in a human-readable format. +// Copied from cmd/bundle/deploy.go +func renderDiagnostics(w io.Writer, b *bundle.Bundle, diags diag.Diagnostics) error { + renderOpts := render.RenderOptions{RenderSummaryTable: false} + err := render.RenderDiagnostics(w, b, diags, renderOpts) + if err != nil { + return fmt.Errorf("failed to render output: %w", err) + } + + if diags.HasError() { + return root.ErrAlreadyPrinted + } + + return nil +} diff --git a/cmd/pipelines/variables.go b/cmd/pipelines/variables.go new file mode 100644 index 0000000000..785a3d73de --- /dev/null +++ b/cmd/pipelines/variables.go @@ -0,0 +1,10 @@ +package pipelines + +import ( + "github.com/spf13/cobra" +) + +// Copied from cmd/bundle/variables.go +func initVariableFlag(cmd *cobra.Command) { + cmd.PersistentFlags().StringSlice("var", []string{}, `set values for variables defined in project config. Example: --var="foo=bar"`) +}