diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 47f5cf1882..c1f7fd6bc9 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,5 +9,6 @@ ### Dependency updates ### Bundles +* Add new Lakeflow Pipelines support for bundle generate ([#3568](https://github.com/databricks/cli/pull/3568)) ### API Changes diff --git a/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml b/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml new file mode 100644 index 0000000000..e092fd5ed6 --- /dev/null +++ b/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/generate/lakeflow_pipelines/out/config/out.pipeline.yml b/acceptance/bundle/generate/lakeflow_pipelines/out/config/out.pipeline.yml new file mode 100644 index 0000000000..479fe57f00 --- /dev/null +++ b/acceptance/bundle/generate/lakeflow_pipelines/out/config/out.pipeline.yml @@ -0,0 +1,10 @@ +resources: + pipelines: + out: + name: lakeflow-pipeline + libraries: + - glob: + include: ../pipeline/transformations/** + - glob: + include: /Workspace/Users/foo@databricks.com/another/** + root_path: ../pipeline diff --git a/acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/explorations/1.py b/acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/explorations/1.py new file mode 100644 index 0000000000..7df869a15e --- /dev/null +++ b/acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/explorations/1.py @@ -0,0 +1 @@ +print("Hello, World!") diff --git a/acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/transformations/1.py b/acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/transformations/1.py new file mode 100644 index 0000000000..7df869a15e --- /dev/null +++ b/acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/transformations/1.py @@ -0,0 +1 @@ +print("Hello, World!") diff --git a/acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/transformations/2.py b/acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/transformations/2.py new file mode 100644 index 0000000000..7df869a15e --- /dev/null +++ b/acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/transformations/2.py @@ -0,0 +1 @@ +print("Hello, World!") diff --git a/acceptance/bundle/generate/lakeflow_pipelines/output.txt b/acceptance/bundle/generate/lakeflow_pipelines/output.txt new file mode 100644 index 0000000000..b4c4e3d60a --- /dev/null +++ b/acceptance/bundle/generate/lakeflow_pipelines/output.txt @@ -0,0 +1,4 @@ +Pipeline configuration successfully saved to out/config/out.pipeline.yml +File successfully saved to out/pipeline/explorations/1.py +File successfully saved to out/pipeline/transformations/1.py +File successfully saved to out/pipeline/transformations/2.py diff --git a/acceptance/bundle/generate/lakeflow_pipelines/pipeline.json b/acceptance/bundle/generate/lakeflow_pipelines/pipeline.json new file mode 100644 index 0000000000..0254d7b549 --- /dev/null +++ b/acceptance/bundle/generate/lakeflow_pipelines/pipeline.json @@ -0,0 +1,16 @@ +{ + "name": "lakeflow-pipeline", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline/transformations/**" + } + }, + { + "glob": { + "include": "/Workspace/Users/foo@databricks.com/another/**" + } + } + ], + "root_path": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline" +} diff --git a/acceptance/bundle/generate/lakeflow_pipelines/script b/acceptance/bundle/generate/lakeflow_pipelines/script new file mode 100644 index 0000000000..f4b7257ecf --- /dev/null +++ b/acceptance/bundle/generate/lakeflow_pipelines/script @@ -0,0 +1,9 @@ +export PIPELINE_ID=$($CLI pipelines create --json @pipeline.json | jq -r .pipeline_id) + +$CLI bundle generate pipeline --existing-pipeline-id ${PIPELINE_ID} --config-dir out/config --key out --source-dir out/pipeline > out.stdout 2> out.stderr + +# Combine stdout and stderr, then sort only the "File successfully saved" lines +cat out.stdout out.stderr > out.txt +grep -v "^File successfully saved" out.txt +grep "^File successfully saved" out.txt | sort +rm out.txt out.stdout out.stderr diff --git a/acceptance/bundle/generate/lakeflow_pipelines/test.toml b/acceptance/bundle/generate/lakeflow_pipelines/test.toml new file mode 100644 index 0000000000..0b39960b4b --- /dev/null +++ b/acceptance/bundle/generate/lakeflow_pipelines/test.toml @@ -0,0 +1,37 @@ +[[Server]] +Pattern = "GET /api/2.0/workspace/export" +Response.Body = ''' +print("Hello, World!") +''' + +[[Repls]] +Old = '\\' +New = '/' + +[[Server]] +Pattern = "GET /api/2.0/workspace/list" +Response.Body = ''' +{ + "objects": [ + { + "path": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline/transformations/1.py", + "object_type": "FILE" + }, + { + "path": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline/transformations/2.py", + "object_type": "FILE" + }, + { + "path": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline/explorations/1.py", + "object_type": "FILE" + } + ] +} +''' + +[[Server]] +Pattern = "GET /api/2.0/workspace/get-status" +Response.Body = ''' +{ +} +''' diff --git a/bundle/generate/pipeline.go b/bundle/generate/pipeline.go index ba4aedfa02..c85e6f88b5 100644 --- a/bundle/generate/pipeline.go +++ b/bundle/generate/pipeline.go @@ -1,6 +1,8 @@ package generate import ( + "strings" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -8,9 +10,21 @@ import ( var pipelineOrder = yamlsaver.NewOrder([]string{"name", "clusters", "configuration", "libraries"}) -func ConvertPipelineToValue(pipeline *pipelines.PipelineSpec) (dyn.Value, error) { - value := make(map[string]dyn.Value) +func ConvertPipelineToValue(pipeline *pipelines.PipelineSpec, rootPath, remoteRootPath string) (dyn.Value, error) { + if pipeline.RootPath != "" { + pipeline.RootPath = rootPath + } + if pipeline.Libraries != nil { + for i := range pipeline.Libraries { + lib := &pipeline.Libraries[i] + if lib.Glob != nil { + lib.Glob.Include = strings.ReplaceAll(lib.Glob.Include, remoteRootPath, rootPath) + } + } + } + + value := make(map[string]dyn.Value) // We ignore the following fields: // - id: this is a read-only field // - storage: changes to this field are rare because changing the storage recreates pipeline-related resources diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go index 9910efc32f..47030a24ec 100644 --- a/cmd/bundle/generate/pipeline.go +++ b/cmd/bundle/generate/pipeline.go @@ -80,7 +80,22 @@ like catalogs, schemas, and compute configurations per target.`, } } - v, err := generate.ConvertPipelineToValue(pipeline.Spec) + // If the root path is set, we need to download the files from the root path + remoteRootPath := pipeline.Spec.RootPath + if pipeline.Spec.RootPath != "" { + err := downloader.MarkDirectoryForDownload(ctx, &pipeline.Spec.RootPath) + if err != nil { + return err + } + } + + // Making sure the root path is relative to the config directory. + rel, err := filepath.Rel(configDir, sourceDir) + if err != nil { + return err + } + + v, err := generate.ConvertPipelineToValue(pipeline.Spec, filepath.ToSlash(rel), remoteRootPath) if err != nil { return err }