diff --git a/.wsignore b/.wsignore index 56898bdba3..245b66b555 100644 --- a/.wsignore +++ b/.wsignore @@ -25,7 +25,6 @@ acceptance/bundle/artifacts/shell/err-sh/output.txt acceptance/bundle/templates-machinery/helpers-error/output.txt # "bundle deploy" with apps has trailing whitespace: -acceptance/bundle/apps/config_section/output.txt integration/bundle/testdata/apps/bundle_deploy.txt # Extra whitespace in test output: diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 22aef9f5a6..2cfb16d1e9 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,6 +9,7 @@ ### Dependency updates ### Bundles +* Remove resources.apps.config section ([#3680](https://github.com/databricks/cli/pull/3680)) * Prompt for serverless compute in `dbt-sql` template (defaults to `yes`) ([#3668](https://github.com/databricks/cli/pull/3668)) ### API Changes diff --git a/acceptance/bundle/apps/config_section/app/app.py b/acceptance/bundle/apps/config_section/app/app.py deleted file mode 100644 index f1a18139c8..0000000000 --- a/acceptance/bundle/apps/config_section/app/app.py +++ /dev/null @@ -1 +0,0 @@ -print("Hello world!") diff --git a/acceptance/bundle/apps/config_section/databricks.yml b/acceptance/bundle/apps/config_section/databricks.yml deleted file mode 100644 index 2662d75f00..0000000000 --- a/acceptance/bundle/apps/config_section/databricks.yml +++ /dev/null @@ -1,9 +0,0 @@ -resources: - apps: - myapp: - name: myapp - source_code_path: ./app - config: - command: - - python - - app.py diff --git a/acceptance/bundle/apps/config_section/out.app.yml.txt b/acceptance/bundle/apps/config_section/out.app.yml.txt deleted file mode 100644 index 6d74c282a2..0000000000 --- a/acceptance/bundle/apps/config_section/out.app.yml.txt +++ /dev/null @@ -1,5 +0,0 @@ -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml", - "raw_body": "command:\n - python\n - app.py\n" -} diff --git a/acceptance/bundle/apps/config_section/out.test.toml b/acceptance/bundle/apps/config_section/out.test.toml deleted file mode 100644 index e092fd5ed6..0000000000 --- a/acceptance/bundle/apps/config_section/out.test.toml +++ /dev/null @@ -1,5 +0,0 @@ -Local = true -Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/apps/config_section/output.txt b/acceptance/bundle/apps/config_section/output.txt deleted file mode 100644 index 32776248d0..0000000000 --- a/acceptance/bundle/apps/config_section/output.txt +++ /dev/null @@ -1,23 +0,0 @@ - ->>> [CLI] bundle validate -Warning: App config section detected - -remove 'config' from app resource 'myapp' section and use app.yml file in the root of this app instead - -Name: test-bundle -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Found 1 warning - ->>> [CLI] bundle deploy -Warning: App config section detected - -remove 'config' from app resource 'myapp' section and use app.yml file in the root of this app instead - -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! diff --git a/acceptance/bundle/apps/config_section/script b/acceptance/bundle/apps/config_section/script deleted file mode 100644 index b80cd910c0..0000000000 --- a/acceptance/bundle/apps/config_section/script +++ /dev/null @@ -1,4 +0,0 @@ -trace $CLI bundle validate -trace $CLI bundle deploy -jq 'select(.path | test("default/files/app/app.yml"))' out.requests.txt > out.app.yml.txt -rm out.requests.txt diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 513ecea1f1..874b95e3db 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -74,8 +74,6 @@ resources.apps.*.budget_policy_id string ALL resources.apps.*.compute_status *apps.ComputeStatus ALL resources.apps.*.compute_status.message string ALL resources.apps.*.compute_status.state apps.ComputeState ALL -resources.apps.*.config map[string]any INPUT -resources.apps.*.config.* any INPUT resources.apps.*.create_time string ALL resources.apps.*.creator string ALL resources.apps.*.default_source_code_path string ALL diff --git a/bundle/apps/interpolate_variables.go b/bundle/apps/interpolate_variables.go deleted file mode 100644 index da7dadf471..0000000000 --- a/bundle/apps/interpolate_variables.go +++ /dev/null @@ -1,45 +0,0 @@ -package apps - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/dynvar" -) - -type interpolateVariables struct{} - -func (i *interpolateVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - pattern := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("apps"), - dyn.AnyKey(), - dyn.Key("config"), - ) - - err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(root, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) { - key, ok := terraform.TerraformToGroupName[path[0].Key()] - if ok { - path = dyn.NewPath(dyn.Key("resources"), dyn.Key(key)).Append(path[1:]...) - } - - return dyn.GetByPath(root, path) - }) - }) - }) - - return diag.FromErr(err) -} - -func (i *interpolateVariables) Name() string { - return "apps.InterpolateVariables" -} - -func InterpolateVariables() bundle.Mutator { - return &interpolateVariables{} -} diff --git a/bundle/apps/interpolate_variables_test.go b/bundle/apps/interpolate_variables_test.go deleted file mode 100644 index 05fc26dd95..0000000000 --- a/bundle/apps/interpolate_variables_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package apps - -import ( - "context" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/databricks-sdk-go/service/apps" - "github.com/stretchr/testify/require" -) - -func TestAppInterpolateVariables(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Apps: map[string]*resources.App{ - "my_app_1": { - App: apps.App{ - Name: "my_app_1", - }, - Config: map[string]any{ - "command": []string{"echo", "hello"}, - "env": []map[string]string{ - {"name": "JOB_ID", "value": "${databricks_job.my_job.id}"}, - }, - }, - }, - "my_app_2": { - App: apps.App{ - Name: "my_app_2", - }, - }, - }, - Jobs: map[string]*resources.Job{ - "my_job": { - BaseResource: resources.BaseResource{ID: "123"}, - }, - }, - }, - }, - } - - diags := bundle.Apply(context.Background(), b, InterpolateVariables()) - require.Empty(t, diags) - require.Equal(t, []any{map[string]any{"name": "JOB_ID", "value": "123"}}, b.Config.Resources.Apps["my_app_1"].Config["env"]) - require.Nil(t, b.Config.Resources.Apps["my_app_2"].Config) -} diff --git a/bundle/apps/upload_config.go b/bundle/apps/upload_config.go deleted file mode 100644 index 82d4a76301..0000000000 --- a/bundle/apps/upload_config.go +++ /dev/null @@ -1,93 +0,0 @@ -package apps - -import ( - "bytes" - "context" - "fmt" - "path" - "strings" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/deploy" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/filer" - "github.com/databricks/cli/libs/logdiag" - "golang.org/x/sync/errgroup" - - "gopkg.in/yaml.v3" -) - -type uploadConfig struct { - filerFactory deploy.FilerFactory -} - -func (u *uploadConfig) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - errGroup, ctx := errgroup.WithContext(ctx) - - for key, app := range b.Config.Resources.Apps { - // If the app has a config, we need to deploy it first. - // It means we need to write app.yml file with the content of the config field - // to the remote source code path of the app. - if app.Config != nil { - appPath := strings.TrimPrefix(app.SourceCodePath, b.Config.Workspace.FilePath) - - buf, err := configToYaml(app) - if err != nil { - return diag.FromErr(err) - } - - f, err := u.filerFactory(b) - if err != nil { - return diag.FromErr(err) - } - - errGroup.Go(func() error { - err := f.Write(ctx, path.Join(appPath, "app.yml"), buf, filer.OverwriteIfExists) - if err != nil { - logdiag.LogDiag(ctx, diag.Diagnostic{ - Severity: diag.Error, - Summary: "Failed to save config", - Detail: fmt.Sprintf("Failed to write %s file: %s", path.Join(app.SourceCodePath, "app.yml"), err), - Locations: b.Config.GetLocations("resources.apps." + key), - }) - } - return nil - }) - } - } - - if err := errGroup.Wait(); err != nil { - logdiag.LogError(ctx, err) - } - - return nil -} - -// Name implements bundle.Mutator. -func (u *uploadConfig) Name() string { - return "apps:UploadConfig" -} - -func UploadConfig() bundle.Mutator { - return &uploadConfig{ - filerFactory: func(b *bundle.Bundle) (filer.Filer, error) { - return filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.FilePath) - }, - } -} - -func configToYaml(app *resources.App) (*bytes.Buffer, error) { - buf := bytes.NewBuffer(nil) - enc := yaml.NewEncoder(buf) - enc.SetIndent(2) - - err := enc.Encode(app.Config) - defer enc.Close() - - if err != nil { - return nil, fmt.Errorf("failed to encode app config to yaml: %w", err) - } - - return buf, nil -} diff --git a/bundle/apps/upload_config_test.go b/bundle/apps/upload_config_test.go deleted file mode 100644 index cf481336cb..0000000000 --- a/bundle/apps/upload_config_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package apps - -import ( - "bytes" - "context" - "os" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/internal/bundletest" - mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/filer" - "github.com/databricks/cli/libs/vfs" - "github.com/databricks/databricks-sdk-go/service/apps" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -func TestAppUploadConfig(t *testing.T) { - root := t.TempDir() - err := os.MkdirAll(filepath.Join(root, "my_app"), 0o700) - require.NoError(t, err) - - b := &bundle.Bundle{ - BundleRootPath: root, - SyncRootPath: root, - SyncRoot: vfs.MustNew(root), - Config: config.Root{ - Workspace: config.Workspace{ - RootPath: "/Workspace/Users/foo@bar.com/", - }, - Resources: config.Resources{ - Apps: map[string]*resources.App{ - "my_app": { - App: apps.App{ - Name: "my_app", - }, - SourceCodePath: "./my_app", - Config: map[string]any{ - "command": []string{"echo", "hello"}, - "env": []map[string]string{ - {"name": "MY_APP", "value": "my value"}, - }, - }, - }, - }, - }, - }, - } - - mockFiler := mockfiler.NewMockFiler(t) - mockFiler.EXPECT().Write(mock.Anything, "my_app/app.yml", bytes.NewBufferString(`command: - - echo - - hello -env: - - name: MY_APP - value: my value -`), filer.OverwriteIfExists).Return(nil) - - u := uploadConfig{ - filerFactory: func(b *bundle.Bundle) (filer.Filer, error) { - return mockFiler, nil - }, - } - - bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(root, "databricks.yml")}}) - - diags := bundle.ApplySeq(context.Background(), b, mutator.TranslatePaths(), &u) - require.NoError(t, diags.Error()) -} diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index 4e743c00a9..33de3cf77e 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -34,14 +34,6 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics }) } usedSourceCodePaths[app.SourceCodePath] = key - - if app.Config != nil { - diags = append(diags, diag.Diagnostic{ - Severity: diag.Warning, - Summary: "App config section detected", - Detail: fmt.Sprintf("remove 'config' from app resource '%s' section and use app.yml file in the root of this app instead", key), - }) - } } return diags diff --git a/bundle/config/mutator/log_resource_references_test.go b/bundle/config/mutator/log_resource_references_test.go index d91885cb0a..c22b50e9ca 100644 --- a/bundle/config/mutator/log_resource_references_test.go +++ b/bundle/config/mutator/log_resource_references_test.go @@ -28,13 +28,6 @@ func TestConvertReferenceToMetric_Table(t *testing.T) { "niljob": nil, }, - Apps: map[string]*cres.App{ - "app1": { - Config: map[string]any{ - "k": "v", - }, - }, - }, }, } @@ -73,11 +66,6 @@ func TestConvertReferenceToMetric_Table(t *testing.T) { ref: "resources.jobs", want: "", }, - { - name: "mapkey censor on app config", - ref: "resources.apps.app1.config.foo", - want: "resref_apps.config.*", - }, { name: "nil job pointer yields plain id", ref: "resources.jobs.niljob.id", diff --git a/bundle/config/resources/apps.go b/bundle/config/resources/apps.go index 177c242e5b..875ecfe476 100644 --- a/bundle/config/resources/apps.go +++ b/bundle/config/resources/apps.go @@ -30,12 +30,6 @@ type App struct { // on local disk and to the corresponding workspace path during app deployment. SourceCodePath string `json:"source_code_path"` - // Config is an optional field which allows configuring the app following Databricks app configuration format like in app.yml. - // When this field is set, DABs read the configuration set in this field and write - // it to app.yml in the root of the source code folder in Databricks workspace. - // If there's app.yml defined locally, DABs will raise an error. - Config map[string]any `json:"config,omitempty"` - Permissions []AppPermission `json:"permissions,omitempty"` } diff --git a/bundle/deploy/terraform/tfdyn/convert_app_test.go b/bundle/deploy/terraform/tfdyn/convert_app_test.go index f95a6419a5..662c172fb9 100644 --- a/bundle/deploy/terraform/tfdyn/convert_app_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_app_test.go @@ -16,9 +16,6 @@ import ( func TestConvertApp(t *testing.T) { src := resources.App{ SourceCodePath: "./app", - Config: map[string]any{ - "command": []string{"python", "app.py"}, - }, App: apps.App{ Name: "app_id", Description: "app description", @@ -101,9 +98,6 @@ func TestConvertApp(t *testing.T) { func TestConvertAppWithNoDescription(t *testing.T) { src := resources.App{ SourceCodePath: "./app", - Config: map[string]any{ - "command": []string{"python", "app.py"}, - }, App: apps.App{ Name: "app_id", Resources: []apps.AppResource{ diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 40d930b6f4..ea012d057e 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -5,7 +5,6 @@ import ( "errors" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/apps" "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deploy" @@ -114,13 +113,6 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) { bundle.ApplySeqContext(ctx, b, statemgmt.Load(), - - // TODO: this does terraform specific transformation. - apps.InterpolateVariables(), - - // TODO: this should either be part of app resource or separate AppConfig resource that depends on main resource. - apps.UploadConfig(), - metadata.Compute(), metadata.Upload(), ) diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go index 0d3d325d93..d1ca6c0adc 100644 --- a/bundle/run/app_test.go +++ b/bundle/run/app_test.go @@ -59,12 +59,6 @@ func setupBundle(t *testing.T) (context.Context, *bundle.Bundle, *mocks.MockWork Name: "my_app", }, SourceCodePath: "./my_app", - Config: map[string]any{ - "command": []string{"echo", "hello"}, - "env": []map[string]string{ - {"name": "MY_APP", "value": "my value"}, - }, - }, }, }, }, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 6a64910fcd..769bb94427 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -80,9 +80,6 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus", "doNotSuggest": true }, - "config": { - "$ref": "#/$defs/map/interface" - }, "create_time": { "description": "The creation time of the app. Formatted timestamp in ISO 6801.", "$ref": "#/$defs/string", @@ -9634,20 +9631,6 @@ } } }, - "interface": { - "oneOf": [ - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/interface" - } - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" - } - ] - }, "string": { "oneOf": [ {