From 06b8ac44bb68c4d4c9d81d9c2592edb547fddb2b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 21 Nov 2024 17:21:25 +0100 Subject: [PATCH 1/6] Added support for Databricks Apps in DABs --- bundle/config/mutator/apply_presets_test.go | 57 +++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go index c26f203832..2af54e74e2 100644 --- a/bundle/config/mutator/apply_presets_test.go +++ b/bundle/config/mutator/apply_presets_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" @@ -482,3 +483,59 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) { }) } } + +func TestApplyPresetsPrefixForApps(t *testing.T) { + tests := []struct { + name string + prefix string + app *resources.App + want string + }{ + { + name: "add prefix to app", + prefix: "[prefix] ", + app: &resources.App{ + App: &apps.App{ + Name: "app1", + }, + }, + want: "prefix-app1", + }, + { + name: "add empty prefix to app", + prefix: "", + app: &resources.App{ + App: &apps.App{ + Name: "app1", + }, + }, + want: "app1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Apps: map[string]*resources.App{ + "app1": tt.app, + }, + }, + Presets: config.Presets{ + NamePrefix: tt.prefix, + }, + }, + } + + ctx := context.Background() + diag := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + if diag.HasError() { + t.Fatalf("unexpected error: %v", diag) + } + + require.Equal(t, tt.want, b.Config.Resources.Apps["app1"].Name) + }) + } +} From 64bcb0a5f693dc221cf3f954ec42873cb145dd50 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 29 Nov 2024 15:54:42 +0100 Subject: [PATCH 2/6] Added support for bundle generate and bind for Apps --- bundle/config/generate/app.go | 37 +++++++++ bundle/config/resources.go | 7 ++ cmd/bundle/generate.go | 1 + cmd/bundle/generate/app.go | 151 ++++++++++++++++++++++++++++++++++ cmd/bundle/generate/utils.go | 32 +++++++ 5 files changed, 228 insertions(+) create mode 100644 bundle/config/generate/app.go create mode 100644 cmd/bundle/generate/app.go diff --git a/bundle/config/generate/app.go b/bundle/config/generate/app.go new file mode 100644 index 0000000000..ce4fe8d623 --- /dev/null +++ b/bundle/config/generate/app.go @@ -0,0 +1,37 @@ +package generate + +import ( + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/apps" +) + +func ConvertAppToValue(app *apps.App, sourceCodePath string, appConfig map[string]interface{}) (dyn.Value, error) { + ac, err := convert.FromTyped(appConfig, dyn.NilValue) + if err != nil { + return dyn.NilValue, err + } + + ar, err := convert.FromTyped(app.Resources, dyn.NilValue) + if err != nil { + return dyn.NilValue, err + } + + // The majority of fields of the app struct are read-only. + // We copy the relevant fields manually. + dv := map[string]dyn.Value{ + "name": dyn.NewValue(app.Name, []dyn.Location{{Line: 1}}), + "description": dyn.NewValue(app.Description, []dyn.Location{{Line: 2}}), + "source_code_path": dyn.NewValue(sourceCodePath, []dyn.Location{{Line: 3}}), + } + + if ac.Kind() != dyn.KindNil { + dv["config"] = ac.WithLocations([]dyn.Location{{Line: 4}}) + } + + if ar.Kind() != dyn.KindNil { + dv["resources"] = ar.WithLocations([]dyn.Location{{Line: 5}}) + } + + return dyn.V(dv), nil +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index bf81b92958..b06a44b4dd 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -99,12 +99,19 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) found = append(found, r.Jobs[k]) } } + for k := range r.Pipelines { if k == key { found = append(found, r.Pipelines[k]) } } + for k := range r.Apps { + if k == key { + found = append(found, r.Apps[k]) + } + } + if len(found) == 0 { return nil, fmt.Errorf("no such resource: %s", key) } diff --git a/cmd/bundle/generate.go b/cmd/bundle/generate.go index 7dea19ff9d..d09c6feb43 100644 --- a/cmd/bundle/generate.go +++ b/cmd/bundle/generate.go @@ -17,6 +17,7 @@ func newGenerateCommand() *cobra.Command { cmd.AddCommand(generate.NewGenerateJobCommand()) cmd.AddCommand(generate.NewGeneratePipelineCommand()) cmd.AddCommand(generate.NewGenerateDashboardCommand()) + cmd.AddCommand(generate.NewGenerateAppCommand()) cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`) return cmd } diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go new file mode 100644 index 0000000000..a10680dffb --- /dev/null +++ b/cmd/bundle/generate/app.go @@ -0,0 +1,151 @@ +package generate + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle/config/generate" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/apps" + "github.com/spf13/cobra" + + "gopkg.in/yaml.v3" +) + +func NewGenerateAppCommand() *cobra.Command { + var configDir string + var sourceDir string + var appName string + var force bool + + cmd := &cobra.Command{ + Use: "app", + Short: "Generate bundle configuration for a Databricks app", + } + + cmd.Flags().StringVar(&appName, "existing-app-name", "", `App name to generate config for`) + cmd.MarkFlagRequired("existing-app-name") + + wd, err := os.Getwd() + if err != nil { + wd = "." + } + + cmd.Flags().StringVarP(&configDir, "config-dir", "d", filepath.Join(wd, "resources"), `Directory path where the output bundle config will be stored`) + cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", filepath.Join(wd, "src", "app"), `Directory path where the app files will be stored`) + cmd.Flags().BoolVarP(&force, "force", "f", false, `Force overwrite existing files in the output directory`) + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + b, diags := root.MustConfigureBundle(cmd) + if err := diags.Error(); err != nil { + return diags.Error() + } + + w := b.WorkspaceClient() + cmdio.LogString(ctx, fmt.Sprintf("Loading app '%s' configuration", appName)) + app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: appName}) + if err != nil { + return err + } + + downloader := newDownloader(w, sourceDir, configDir) + + sourceCodePath := app.DefaultSourceCodePath + downloader.markDirectoryForDownload(ctx, &sourceCodePath) + + appConfig, err := getAppConfig(ctx, app, w) + if err != nil { + return fmt.Errorf("failed to get app config: %w", err) + } + + v, err := generate.ConvertAppToValue(app, sourceCodePath, appConfig) + if err != nil { + return err + } + + appKey := cmd.Flag("key").Value.String() + if appKey == "" { + appKey = textutil.NormalizeString(app.Name) + } + + result := map[string]dyn.Value{ + "resources": dyn.V(map[string]dyn.Value{ + "apps": dyn.V(map[string]dyn.Value{ + appKey: v, + }), + }), + } + + // If there are app.yaml or app.yml files in the source code path, they will be downloaded but we don't want to include them in the bundle. + // We include this configuration inline, so we need to remove these files. + for _, configFile := range []string{"app.yml", "app.yaml"} { + delete(downloader.files, filepath.Join(sourceDir, configFile)) + } + + err = downloader.FlushToDisk(ctx, force) + if err != nil { + return err + } + + filename := filepath.Join(configDir, fmt.Sprintf("%s.app.yml", appKey)) + + saver := yamlsaver.NewSaver() + err = saver.SaveAsYAML(result, filename, force) + if err != nil { + return err + } + + cmdio.LogString(ctx, fmt.Sprintf("App configuration successfully saved to %s", filename)) + return nil + } + + return cmd +} + +func getAppConfig(ctx context.Context, app *apps.App, w *databricks.WorkspaceClient) (map[string]interface{}, error) { + sourceCodePath := app.DefaultSourceCodePath + + f, err := filer.NewWorkspaceFilesClient(w, sourceCodePath) + if err != nil { + return nil, err + } + + // The app config is stored in app.yml or app.yaml file in the source code path. + configFileNames := []string{"app.yml", "app.yaml"} + for _, configFile := range configFileNames { + r, err := f.Read(ctx, configFile) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + return nil, err + } + defer r.Close() + + cmdio.LogString(ctx, fmt.Sprintf("Reading app configuration from %s", configFile)) + content, err := io.ReadAll(r) + + var appConfig map[string]interface{} + err = yaml.Unmarshal(content, &appConfig) + if err != nil { + cmdio.LogString(ctx, fmt.Sprintf("Failed to parse app configuration:\n%s\nerr: %v", string(content), err)) + return nil, nil + } + + return appConfig, nil + } + + return nil, nil +} diff --git a/cmd/bundle/generate/utils.go b/cmd/bundle/generate/utils.go index 8e3764e352..65bf8ad2c7 100644 --- a/cmd/bundle/generate/utils.go +++ b/cmd/bundle/generate/utils.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/databricks/databricks-sdk-go/service/workspace" "golang.org/x/sync/errgroup" ) @@ -63,6 +64,37 @@ func (n *downloader) markFileForDownload(ctx context.Context, filePath *string) return nil } +func (n *downloader) markDirectoryForDownload(ctx context.Context, dirPath *string) error { + _, err := n.w.Workspace.GetStatusByPath(ctx, *dirPath) + if err != nil { + return err + } + + objects, err := n.w.Workspace.RecursiveList(ctx, *dirPath) + if err != nil { + return err + } + + for _, obj := range objects { + if obj.ObjectType == workspace.ObjectTypeDirectory { + continue + } + + err := n.markFileForDownload(ctx, &obj.Path) + if err != nil { + return err + } + } + + rel, err := filepath.Rel(n.configDir, n.sourceDir) + if err != nil { + return err + } + + *dirPath = rel + return nil +} + func (n *downloader) markNotebookForDownload(ctx context.Context, notebookPath *string) error { info, err := n.w.Workspace.GetStatusByPath(ctx, *notebookPath) if err != nil { From e31306a31a12203a63948bb483a69a125fda977d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 2 Dec 2024 16:07:03 +0100 Subject: [PATCH 3/6] added missing err check --- cmd/bundle/generate/app.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go index a10680dffb..2c0b658325 100644 --- a/cmd/bundle/generate/app.go +++ b/cmd/bundle/generate/app.go @@ -136,6 +136,9 @@ func getAppConfig(ctx context.Context, app *apps.App, w *databricks.WorkspaceCli cmdio.LogString(ctx, fmt.Sprintf("Reading app configuration from %s", configFile)) content, err := io.ReadAll(r) + if err != nil { + return nil, err + } var appConfig map[string]interface{} err = yaml.Unmarshal(content, &appConfig) From fa9278ebbec9e6c5835ab4fcb28249da3aecbc7f Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 18 Dec 2024 12:10:55 +0100 Subject: [PATCH 4/6] fixed generating to different source folder --- cmd/bundle/generate/app.go | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go index 2c0b658325..e706b936ea 100644 --- a/cmd/bundle/generate/app.go +++ b/cmd/bundle/generate/app.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "io/fs" - "os" "path/filepath" "github.com/databricks/cli/bundle/config/generate" @@ -37,13 +36,8 @@ func NewGenerateAppCommand() *cobra.Command { cmd.Flags().StringVar(&appName, "existing-app-name", "", `App name to generate config for`) cmd.MarkFlagRequired("existing-app-name") - wd, err := os.Getwd() - if err != nil { - wd = "." - } - - cmd.Flags().StringVarP(&configDir, "config-dir", "d", filepath.Join(wd, "resources"), `Directory path where the output bundle config will be stored`) - cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", filepath.Join(wd, "src", "app"), `Directory path where the app files will be stored`) + cmd.Flags().StringVarP(&configDir, "config-dir", "d", filepath.Join("resources"), `Directory path where the output bundle config will be stored`) + cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", filepath.Join("src", "app"), `Directory path where the app files will be stored`) cmd.Flags().BoolVarP(&force, "force", "f", false, `Force overwrite existing files in the output directory`) cmd.RunE = func(cmd *cobra.Command, args []string) error { @@ -60,17 +54,35 @@ func NewGenerateAppCommand() *cobra.Command { return err } + // Making sure the config directory and source directory are absolute paths. + if !filepath.IsAbs(configDir) { + configDir = filepath.Join(b.BundleRootPath, configDir) + } + + if !filepath.IsAbs(sourceDir) { + sourceDir = filepath.Join(b.BundleRootPath, sourceDir) + } + downloader := newDownloader(w, sourceDir, configDir) sourceCodePath := app.DefaultSourceCodePath - downloader.markDirectoryForDownload(ctx, &sourceCodePath) + err = downloader.markDirectoryForDownload(ctx, &sourceCodePath) + if err != nil { + return err + } appConfig, err := getAppConfig(ctx, app, w) if err != nil { return fmt.Errorf("failed to get app config: %w", err) } - v, err := generate.ConvertAppToValue(app, sourceCodePath, appConfig) + // Making sure the source code path is relative to the config directory. + rel, err := filepath.Rel(configDir, sourceDir) + if err != nil { + return err + } + + v, err := generate.ConvertAppToValue(app, filepath.ToSlash(rel), appConfig) if err != nil { return err } From 72f70bf87206609001a6281b6da6684a68a71134 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 18 Dec 2024 13:07:15 +0100 Subject: [PATCH 5/6] removed the test --- bundle/config/mutator/apply_presets_test.go | 57 --------------------- 1 file changed, 57 deletions(-) diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go index 2af54e74e2..c26f203832 100644 --- a/bundle/config/mutator/apply_presets_test.go +++ b/bundle/config/mutator/apply_presets_test.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" @@ -483,59 +482,3 @@ func TestApplyPresetsSourceLinkedDeployment(t *testing.T) { }) } } - -func TestApplyPresetsPrefixForApps(t *testing.T) { - tests := []struct { - name string - prefix string - app *resources.App - want string - }{ - { - name: "add prefix to app", - prefix: "[prefix] ", - app: &resources.App{ - App: &apps.App{ - Name: "app1", - }, - }, - want: "prefix-app1", - }, - { - name: "add empty prefix to app", - prefix: "", - app: &resources.App{ - App: &apps.App{ - Name: "app1", - }, - }, - want: "app1", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Apps: map[string]*resources.App{ - "app1": tt.app, - }, - }, - Presets: config.Presets{ - NamePrefix: tt.prefix, - }, - }, - } - - ctx := context.Background() - diag := bundle.Apply(ctx, b, mutator.ApplyPresets()) - - if diag.HasError() { - t.Fatalf("unexpected error: %v", diag) - } - - require.Equal(t, tt.want, b.Config.Resources.Apps["app1"].Name) - }) - } -} From 2eda8e93f67ef5ffcfa835d10346c08d0a8a42a3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 18 Dec 2024 13:10:59 +0100 Subject: [PATCH 6/6] fix lint --- bundle/config/generate/app.go | 2 +- cmd/bundle/generate/app.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bundle/config/generate/app.go b/bundle/config/generate/app.go index ce4fe8d623..1255d63f89 100644 --- a/bundle/config/generate/app.go +++ b/bundle/config/generate/app.go @@ -6,7 +6,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/apps" ) -func ConvertAppToValue(app *apps.App, sourceCodePath string, appConfig map[string]interface{}) (dyn.Value, error) { +func ConvertAppToValue(app *apps.App, sourceCodePath string, appConfig map[string]any) (dyn.Value, error) { ac, err := convert.FromTyped(appConfig, dyn.NilValue) if err != nil { return dyn.NilValue, err diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go index e706b936ea..c948cf074c 100644 --- a/cmd/bundle/generate/app.go +++ b/cmd/bundle/generate/app.go @@ -126,7 +126,7 @@ func NewGenerateAppCommand() *cobra.Command { return cmd } -func getAppConfig(ctx context.Context, app *apps.App, w *databricks.WorkspaceClient) (map[string]interface{}, error) { +func getAppConfig(ctx context.Context, app *apps.App, w *databricks.WorkspaceClient) (map[string]any, error) { sourceCodePath := app.DefaultSourceCodePath f, err := filer.NewWorkspaceFilesClient(w, sourceCodePath) @@ -152,7 +152,7 @@ func getAppConfig(ctx context.Context, app *apps.App, w *databricks.WorkspaceCli return nil, err } - var appConfig map[string]interface{} + var appConfig map[string]any err = yaml.Unmarshal(content, &appConfig) if err != nil { cmdio.LogString(ctx, fmt.Sprintf("Failed to parse app configuration:\n%s\nerr: %v", string(content), err))