From 75b71555438bf14491efb826af3b65433d901654 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Thu, 3 Apr 2025 15:55:07 +0200 Subject: [PATCH 1/5] Mark deprecated pipeline fields --- NEXT_CHANGELOG.md | 1 + bundle/docsgen/main.go | 3 +++ bundle/internal/annotation/descriptor.go | 1 + bundle/internal/schema/annotations.go | 4 ++++ .../internal/schema/annotations_openapi_overrides.yml | 7 +++++++ bundle/schema/jsonschema.json | 11 ++++++++--- 6 files changed, 24 insertions(+), 3 deletions(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 97396c8d13..9acffdffa2 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,5 +10,6 @@ ### Bundles * Added support for model serving endpoints in deployment bind/unbind commands ([#2634](https://github.com/databricks/cli/pull/2634)) +* Updated json schema for deprecated pipeline fields ([#2646](https://github.com/databricks/cli/pull/2646)) ### API Changes diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go index 6898209857..f873f32b84 100644 --- a/bundle/docsgen/main.go +++ b/bundle/docsgen/main.go @@ -137,6 +137,9 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.Deprecated = true s.DeprecationMessage = a.DeprecationMessage } + if a.Deprecated { + s.Deprecated = true + } } func fillTemplateVariables(s string) string { diff --git a/bundle/internal/annotation/descriptor.go b/bundle/internal/annotation/descriptor.go index afcc0ce9fe..25c228d5a0 100644 --- a/bundle/internal/annotation/descriptor.go +++ b/bundle/internal/annotation/descriptor.go @@ -8,6 +8,7 @@ type Descriptor struct { Enum []any `json:"enum,omitempty"` MarkdownExamples string `json:"markdown_examples,omitempty"` DeprecationMessage string `json:"deprecation_message,omitempty"` + Deprecated bool `json:"deprecated,omitempty"` } const Placeholder = "PLACEHOLDER" diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index 7fac538340..ecc2597840 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -133,6 +133,10 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.DeprecationMessage = a.DeprecationMessage } + if a.Deprecated { + s.Deprecated = true + } + s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription) s.Title = a.Title s.Enum = a.Enum diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index ffd1ba2634..90714fab2e 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -230,6 +230,10 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "run_as": "description": |- PLACEHOLDER + "target": + "deprecation_message": This legacy field is deprecated for pipeline creation in favor of the schema field + "trigger": + "deprecation_message": Use continuous instead github.com/databricks/cli/bundle/config/resources.QualityMonitor: "_": "markdown_description": |- @@ -567,6 +571,9 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: "manual": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary: + "whl": + "deprecated": true github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: "entity_version": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index d5f3e1384f..c8344f35e9 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -768,11 +768,15 @@ }, "target": { "description": "Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "deprecationMessage": "This legacy field is deprecated for pipeline creation in favor of the schema field", + "deprecated": true }, "trigger": { "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger", + "deprecationMessage": "Use continuous instead", + "deprecated": true } }, "additionalProperties": false, @@ -5511,7 +5515,8 @@ }, "whl": { "description": "URI of the whl to be installed.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "deprecated": true } }, "additionalProperties": false From c79d6429945bb9a8157bcce8771dfb46e22b0b05 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Mon, 7 Apr 2025 16:47:34 +0200 Subject: [PATCH 2/5] Update deprecation for 'target' field --- bundle/internal/schema/annotations_openapi_overrides.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 90714fab2e..6c25c05e40 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -231,7 +231,9 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "description": |- PLACEHOLDER "target": - "deprecation_message": This legacy field is deprecated for pipeline creation in favor of the schema field + # should not be marked as deprecated until its possible to switch + # existing pipelines to use 'schema' instead + "deprecated": false "trigger": "deprecation_message": Use continuous instead github.com/databricks/cli/bundle/config/resources.QualityMonitor: From 284951ed3c4182455aba06cd4eaec02ac164c355 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Mon, 7 Apr 2025 16:48:20 +0200 Subject: [PATCH 3/5] Update schema --- bundle/schema/jsonschema.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index c8344f35e9..36b1335587 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -768,9 +768,7 @@ }, "target": { "description": "Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field.", - "$ref": "#/$defs/string", - "deprecationMessage": "This legacy field is deprecated for pipeline creation in favor of the schema field", - "deprecated": true + "$ref": "#/$defs/string" }, "trigger": { "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", From 475d663f9dde4b657c8878a3db9e4ab746e5ce4c Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 8 Apr 2025 13:36:37 +0200 Subject: [PATCH 4/5] Address CR feedback --- bundle/docsgen/main.go | 3 --- bundle/internal/annotation/descriptor.go | 1 - bundle/internal/schema/annotations.go | 4 ---- bundle/internal/schema/annotations_openapi_overrides.yml | 4 ++-- bundle/schema/jsonschema.json | 1 + 5 files changed, 3 insertions(+), 10 deletions(-) diff --git a/bundle/docsgen/main.go b/bundle/docsgen/main.go index f873f32b84..6898209857 100644 --- a/bundle/docsgen/main.go +++ b/bundle/docsgen/main.go @@ -137,9 +137,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.Deprecated = true s.DeprecationMessage = a.DeprecationMessage } - if a.Deprecated { - s.Deprecated = true - } } func fillTemplateVariables(s string) string { diff --git a/bundle/internal/annotation/descriptor.go b/bundle/internal/annotation/descriptor.go index 25c228d5a0..afcc0ce9fe 100644 --- a/bundle/internal/annotation/descriptor.go +++ b/bundle/internal/annotation/descriptor.go @@ -8,7 +8,6 @@ type Descriptor struct { Enum []any `json:"enum,omitempty"` MarkdownExamples string `json:"markdown_examples,omitempty"` DeprecationMessage string `json:"deprecation_message,omitempty"` - Deprecated bool `json:"deprecated,omitempty"` } const Placeholder = "PLACEHOLDER" diff --git a/bundle/internal/schema/annotations.go b/bundle/internal/schema/annotations.go index ecc2597840..7fac538340 100644 --- a/bundle/internal/schema/annotations.go +++ b/bundle/internal/schema/annotations.go @@ -133,10 +133,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) { s.DeprecationMessage = a.DeprecationMessage } - if a.Deprecated { - s.Deprecated = true - } - s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription) s.Title = a.Title s.Enum = a.Enum diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 6c25c05e40..d9d113a286 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -233,7 +233,7 @@ github.com/databricks/cli/bundle/config/resources.Pipeline: "target": # should not be marked as deprecated until its possible to switch # existing pipelines to use 'schema' instead - "deprecated": false + "deprecation_message": "" "trigger": "deprecation_message": Use continuous instead github.com/databricks/cli/bundle/config/resources.QualityMonitor: @@ -575,7 +575,7 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger: PLACEHOLDER github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary: "whl": - "deprecated": true + "deprecation_message": "This field is deprecated" github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput: "entity_version": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 943be10238..204630684e 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -5510,6 +5510,7 @@ "whl": { "description": "URI of the whl to be installed.", "$ref": "#/$defs/string", + "deprecationMessage": "This field is deprecated", "deprecated": true } }, From f596fa708c6a48634e7de03717f99a379df97da5 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 8 Apr 2025 13:37:13 +0200 Subject: [PATCH 5/5] Address CR feedback --- NEXT_CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 59383300da..7f60ad32fc 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -13,6 +13,6 @@ * Added support for model serving endpoints in deployment bind/unbind commands ([#2634](https://github.com/databricks/cli/pull/2634)) * Added include/exclude flags support to bundle sync command ([#2650](https://github.com/databricks/cli/pull/2650)) * Removed pipeline 'deployment' field from jsonschema ([#2653](https://github.com/databricks/cli/pull/2653)) -* Updated json schema for deprecated pipeline fields ([#2646](https://github.com/databricks/cli/pull/2646)) +* Updated JSON schema for deprecated pipeline fields ([#2646](https://github.com/databricks/cli/pull/2646)) ### API Changes