Added new Lakeflow Pipelines support for bundle generate

NEXT_CHANGELOG.md

@@ -9,5 +9,6 @@
 ### Dependency updates
 
 ### Bundles
+* Add new Lakeflow Pipelines support for bundle generate ([#3568](https://github.com/databricks/cli/pull/3568))
 
 ### API Changes

acceptance/bundle/generate/lakeflow_pipelines/out/config/out.pipeline.yml

@@ -0,0 +1,10 @@
+resources:
+  pipelines:
+    out:
+      name: lakeflow-pipeline
+      libraries:
+        - glob:
+            include: ../pipeline/transformations/**
+        - glob:
+            include: /Workspace/Users/foo@databricks.com/another/**
+      root_path: ../pipeline

acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/explorations/1.py

@@ -0,0 +1 @@
+print("Hello, World!")

acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/transformations/1.py

@@ -0,0 +1 @@
+print("Hello, World!")

acceptance/bundle/generate/lakeflow_pipelines/out/pipeline/transformations/2.py

@@ -0,0 +1 @@
+print("Hello, World!")

acceptance/bundle/generate/lakeflow_pipelines/output.txt

@@ -0,0 +1,4 @@
+Pipeline configuration successfully saved to out/config/out.pipeline.yml
+File successfully saved to out/pipeline/explorations/1.py
+File successfully saved to out/pipeline/transformations/1.py
+File successfully saved to out/pipeline/transformations/2.py

acceptance/bundle/generate/lakeflow_pipelines/pipeline.json

@@ -0,0 +1,16 @@
+{
+    "name": "lakeflow-pipeline",
+    "libraries": [
+        {
+            "glob": {
+                "include": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline/transformations/**"
+            }
+        },
+        {
+            "glob": {
+                "include": "/Workspace/Users/foo@databricks.com/another/**"
+            }
+        }
+    ],
+    "root_path": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline"
+}

acceptance/bundle/generate/lakeflow_pipelines/script

@@ -0,0 +1,9 @@
+export PIPELINE_ID=$($CLI pipelines create --json @pipeline.json | jq -r .pipeline_id)
+
+$CLI bundle generate pipeline --existing-pipeline-id ${PIPELINE_ID} --config-dir out/config --key out --source-dir out/pipeline > out.stdout 2> out.stderr
+
+# Combine stdout and stderr, then sort only the "File successfully saved" lines
+cat out.stdout out.stderr > out.txt
+grep -v "^File successfully saved" out.txt
+grep "^File successfully saved" out.txt | sort
+rm out.txt out.stdout out.stderr

acceptance/bundle/generate/lakeflow_pipelines/test.toml

@@ -0,0 +1,37 @@
+[[Server]]
+Pattern = "GET /api/2.0/workspace/export"
+Response.Body = '''
+print("Hello, World!")
+'''
+
+[[Repls]]
+Old = '\\'
+New = '/'
+
+[[Server]]
+Pattern = "GET /api/2.0/workspace/list"
+Response.Body = '''
+{
+    "objects": [
+        {
+            "path": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline/transformations/1.py",
+            "object_type": "FILE"
+        },
+        {
+            "path": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline/transformations/2.py",
+            "object_type": "FILE"
+        },
+        {
+            "path": "/Workspace/Users/tester@databricks.com/lakeflow_pipeline/explorations/1.py",
+            "object_type": "FILE"
+        }
+    ]
+}
+'''
+
+[[Server]]
+Pattern = "GET /api/2.0/workspace/get-status"
+Response.Body = '''
+{
+}
+'''

bundle/generate/pipeline.go

@@ -1,16 +1,30 @@
 package generate
 
 import (
+	"strings"
+
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/dyn/yamlsaver"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
 )
 
 var pipelineOrder = yamlsaver.NewOrder([]string{"name", "clusters", "configuration", "libraries"})
 
-func ConvertPipelineToValue(pipeline *pipelines.PipelineSpec) (dyn.Value, error) {
-	value := make(map[string]dyn.Value)
+func ConvertPipelineToValue(pipeline *pipelines.PipelineSpec, rootPath, remoteRootPath string) (dyn.Value, error) {
+	if pipeline.RootPath != "" {
+		pipeline.RootPath = rootPath
+	}
 
+	if pipeline.Libraries != nil {
+		for i := range pipeline.Libraries {
+			lib := &pipeline.Libraries[i]
+			if lib.Glob != nil {
+				lib.Glob.Include = strings.ReplaceAll(lib.Glob.Include, remoteRootPath, rootPath)
+			}
+		}
+	}
+
+	value := make(map[string]dyn.Value)
 	// We ignore the following fields:
 	// - id: this is a read-only field
 	// - storage: changes to this field are rare because changing the storage recreates pipeline-related resources

cmd/bundle/generate/pipeline.go

@@ -80,7 +80,22 @@ like catalogs, schemas, and compute configurations per target.`,
 			}
 		}
 
-		v, err := generate.ConvertPipelineToValue(pipeline.Spec)
+		// If the root path is set, we need to download the files from the root path
+		remoteRootPath := pipeline.Spec.RootPath
+		if pipeline.Spec.RootPath != "" {
+			err := downloader.MarkDirectoryForDownload(ctx, &pipeline.Spec.RootPath)
+			if err != nil {
+				return err
+			}
+		}
+
+		// Making sure the root path is relative to the config directory.
+		rel, err := filepath.Rel(configDir, sourceDir)
+		if err != nil {
+			return err
+		}
+
+		v, err := generate.ConvertPipelineToValue(pipeline.Spec, filepath.ToSlash(rel), remoteRootPath)
 		if err != nil {
 			return err
 		}