Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions bundle/config/generate/app.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package generate

import (
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/convert"
"github.com/databricks/databricks-sdk-go/service/apps"
)

func ConvertAppToValue(app *apps.App, sourceCodePath string, appConfig map[string]any) (dyn.Value, error) {
ac, err := convert.FromTyped(appConfig, dyn.NilValue)
if err != nil {
return dyn.NilValue, err
}

ar, err := convert.FromTyped(app.Resources, dyn.NilValue)
if err != nil {
return dyn.NilValue, err
}

// The majority of fields of the app struct are read-only.
// We copy the relevant fields manually.
dv := map[string]dyn.Value{
"name": dyn.NewValue(app.Name, []dyn.Location{{Line: 1}}),
"description": dyn.NewValue(app.Description, []dyn.Location{{Line: 2}}),
"source_code_path": dyn.NewValue(sourceCodePath, []dyn.Location{{Line: 3}}),
}

if ac.Kind() != dyn.KindNil {
dv["config"] = ac.WithLocations([]dyn.Location{{Line: 4}})
}

if ar.Kind() != dyn.KindNil {
dv["resources"] = ar.WithLocations([]dyn.Location{{Line: 5}})
}

return dyn.V(dv), nil
}
7 changes: 7 additions & 0 deletions bundle/config/resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,19 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
found = append(found, r.Jobs[k])
}
}

for k := range r.Pipelines {
if k == key {
found = append(found, r.Pipelines[k])
}
}

for k := range r.Apps {
if k == key {
found = append(found, r.Apps[k])
}
}

if len(found) == 0 {
return nil, fmt.Errorf("no such resource: %s", key)
}
Expand Down
1 change: 1 addition & 0 deletions cmd/bundle/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ func newGenerateCommand() *cobra.Command {
cmd.AddCommand(generate.NewGenerateJobCommand())
cmd.AddCommand(generate.NewGeneratePipelineCommand())
cmd.AddCommand(generate.NewGenerateDashboardCommand())
cmd.AddCommand(generate.NewGenerateAppCommand())
cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`)
return cmd
}
166 changes: 166 additions & 0 deletions cmd/bundle/generate/app.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
package generate

import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"path/filepath"

"github.com/databricks/cli/bundle/config/generate"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/yamlsaver"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/textutil"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/service/apps"
"github.com/spf13/cobra"

"gopkg.in/yaml.v3"
)

func NewGenerateAppCommand() *cobra.Command {
var configDir string
var sourceDir string
var appName string
var force bool

cmd := &cobra.Command{
Use: "app",
Short: "Generate bundle configuration for a Databricks app",
}

cmd.Flags().StringVar(&appName, "existing-app-name", "", `App name to generate config for`)
cmd.MarkFlagRequired("existing-app-name")

cmd.Flags().StringVarP(&configDir, "config-dir", "d", filepath.Join("resources"), `Directory path where the output bundle config will be stored`)
cmd.Flags().StringVarP(&sourceDir, "source-dir", "s", filepath.Join("src", "app"), `Directory path where the app files will be stored`)
cmd.Flags().BoolVarP(&force, "force", "f", false, `Force overwrite existing files in the output directory`)

cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
b, diags := root.MustConfigureBundle(cmd)
if err := diags.Error(); err != nil {
return diags.Error()
}

w := b.WorkspaceClient()
cmdio.LogString(ctx, fmt.Sprintf("Loading app '%s' configuration", appName))
app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: appName})
if err != nil {
return err
}

// Making sure the config directory and source directory are absolute paths.
if !filepath.IsAbs(configDir) {
configDir = filepath.Join(b.BundleRootPath, configDir)
}

if !filepath.IsAbs(sourceDir) {
sourceDir = filepath.Join(b.BundleRootPath, sourceDir)
}

downloader := newDownloader(w, sourceDir, configDir)

sourceCodePath := app.DefaultSourceCodePath
err = downloader.markDirectoryForDownload(ctx, &sourceCodePath)
if err != nil {
return err
}

appConfig, err := getAppConfig(ctx, app, w)
if err != nil {
return fmt.Errorf("failed to get app config: %w", err)
}

// Making sure the source code path is relative to the config directory.
rel, err := filepath.Rel(configDir, sourceDir)
if err != nil {
return err
}

v, err := generate.ConvertAppToValue(app, filepath.ToSlash(rel), appConfig)
if err != nil {
return err
}

appKey := cmd.Flag("key").Value.String()
if appKey == "" {
appKey = textutil.NormalizeString(app.Name)
}

result := map[string]dyn.Value{
"resources": dyn.V(map[string]dyn.Value{
"apps": dyn.V(map[string]dyn.Value{
appKey: v,
}),
}),
}

// If there are app.yaml or app.yml files in the source code path, they will be downloaded but we don't want to include them in the bundle.
// We include this configuration inline, so we need to remove these files.
for _, configFile := range []string{"app.yml", "app.yaml"} {
delete(downloader.files, filepath.Join(sourceDir, configFile))
}

err = downloader.FlushToDisk(ctx, force)
if err != nil {
return err
}

filename := filepath.Join(configDir, fmt.Sprintf("%s.app.yml", appKey))

saver := yamlsaver.NewSaver()
err = saver.SaveAsYAML(result, filename, force)
if err != nil {
return err
}

cmdio.LogString(ctx, fmt.Sprintf("App configuration successfully saved to %s", filename))
return nil
}

return cmd
}

func getAppConfig(ctx context.Context, app *apps.App, w *databricks.WorkspaceClient) (map[string]any, error) {
sourceCodePath := app.DefaultSourceCodePath

f, err := filer.NewWorkspaceFilesClient(w, sourceCodePath)
if err != nil {
return nil, err
}

// The app config is stored in app.yml or app.yaml file in the source code path.
configFileNames := []string{"app.yml", "app.yaml"}
for _, configFile := range configFileNames {
r, err := f.Read(ctx, configFile)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
continue
}
return nil, err
}
defer r.Close()

cmdio.LogString(ctx, fmt.Sprintf("Reading app configuration from %s", configFile))
content, err := io.ReadAll(r)
if err != nil {
return nil, err
}

var appConfig map[string]any
err = yaml.Unmarshal(content, &appConfig)
if err != nil {
cmdio.LogString(ctx, fmt.Sprintf("Failed to parse app configuration:\n%s\nerr: %v", string(content), err))
return nil, nil
}

return appConfig, nil
}

return nil, nil
}
32 changes: 32 additions & 0 deletions cmd/bundle/generate/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/databricks/databricks-sdk-go/service/workspace"
"golang.org/x/sync/errgroup"
)

Expand Down Expand Up @@ -63,6 +64,37 @@ func (n *downloader) markFileForDownload(ctx context.Context, filePath *string)
return nil
}

func (n *downloader) markDirectoryForDownload(ctx context.Context, dirPath *string) error {
_, err := n.w.Workspace.GetStatusByPath(ctx, *dirPath)
if err != nil {
return err
}

objects, err := n.w.Workspace.RecursiveList(ctx, *dirPath)
if err != nil {
return err
}

for _, obj := range objects {
if obj.ObjectType == workspace.ObjectTypeDirectory {
continue
}

err := n.markFileForDownload(ctx, &obj.Path)
if err != nil {
return err
}
}

rel, err := filepath.Rel(n.configDir, n.sourceDir)
if err != nil {
return err
}

*dirPath = rel
return nil
}

func (n *downloader) markNotebookForDownload(ctx context.Context, notebookPath *string) error {
info, err := n.w.Workspace.GetStatusByPath(ctx, *notebookPath)
if err != nil {
Expand Down