diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 881fbbd4c5..a9778fef57 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,5 +9,6 @@ ### CLI ### Bundles +- Removed unused fields from resources.models schema: creation\_timestamp, last\_updated\_timestamp, latest\_versions and user\_id. Using them now raises a warning. ### API Changes diff --git a/acceptance/bundle/validate/models/missing_name/databricks.yml b/acceptance/bundle/validate/models/missing_name/databricks.yml new file mode 100644 index 0000000000..c50607ee39 --- /dev/null +++ b/acceptance/bundle/validate/models/missing_name/databricks.yml @@ -0,0 +1,6 @@ +bundle: + name: test-bundle + +resources: + models: + mymodel: {} diff --git a/acceptance/bundle/validate/models/missing_name/output.txt b/acceptance/bundle/validate/models/missing_name/output.txt new file mode 100644 index 0000000000..4f189effdf --- /dev/null +++ b/acceptance/bundle/validate/models/missing_name/output.txt @@ -0,0 +1,7 @@ +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Validation OK! diff --git a/acceptance/bundle/validate/models/missing_name/script b/acceptance/bundle/validate/models/missing_name/script new file mode 100644 index 0000000000..72555b332a --- /dev/null +++ b/acceptance/bundle/validate/models/missing_name/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/acceptance/bundle/validate/models/missing_name/test.toml b/acceptance/bundle/validate/models/missing_name/test.toml new file mode 100644 index 0000000000..c8236f9bf8 --- /dev/null +++ b/acceptance/bundle/validate/models/missing_name/test.toml @@ -0,0 +1 @@ +Badness = "Name is required in the schema but not by DABs" diff --git a/acceptance/bundle/validate/models/user_id/databricks.yml b/acceptance/bundle/validate/models/user_id/databricks.yml new file mode 100644 index 0000000000..61fc174633 --- /dev/null +++ b/acceptance/bundle/validate/models/user_id/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + +resources: + models: + mymodel: + user_id: 123 diff --git a/acceptance/bundle/validate/models/user_id/output.txt b/acceptance/bundle/validate/models/user_id/output.txt new file mode 100644 index 0000000000..8c0d8db730 --- /dev/null +++ b/acceptance/bundle/validate/models/user_id/output.txt @@ -0,0 +1,11 @@ +Warning: unknown field: user_id + at resources.models.mymodel + in databricks.yml:7:7 + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 warning diff --git a/acceptance/bundle/validate/models/user_id/script b/acceptance/bundle/validate/models/user_id/script new file mode 100644 index 0000000000..72555b332a --- /dev/null +++ b/acceptance/bundle/validate/models/user_id/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/bundle/config/mutator/initialize_urls_test.go b/bundle/config/mutator/initialize_urls_test.go index 25ba064f91..fd1f466248 100644 --- a/bundle/config/mutator/initialize_urls_test.go +++ b/bundle/config/mutator/initialize_urls_test.go @@ -43,8 +43,8 @@ func TestInitializeURLs(t *testing.T) { }, Models: map[string]*resources.MlflowModel{ "model1": { - ID: "a model uses its name for identifier", - Model: ml.Model{Name: "a model uses its name for identifier"}, + ID: "a model uses its name for identifier", + CreateModelRequest: ml.CreateModelRequest{Name: "a model uses its name for identifier"}, }, }, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index fbdb5321a9..6f32300631 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -97,7 +97,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle { "experiment2": {Experiment: ml.Experiment{Name: "experiment2"}}, }, Models: map[string]*resources.MlflowModel{ - "model1": {Model: ml.Model{Name: "model1"}}, + "model1": {CreateModelRequest: ml.CreateModelRequest{Name: "model1"}}, }, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ "servingendpoint1": {CreateServingEndpoint: serving.CreateServingEndpoint{Name: "servingendpoint1"}}, diff --git a/bundle/config/resources/mlflow_model.go b/bundle/config/resources/mlflow_model.go index 9c77c5e9bf..a848921b2e 100644 --- a/bundle/config/resources/mlflow_model.go +++ b/bundle/config/resources/mlflow_model.go @@ -28,7 +28,7 @@ type MlflowModel struct { ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` URL string `json:"url,omitempty" bundle:"internal"` - ml.Model + ml.CreateModelRequest } func (s *MlflowModel) UnmarshalJSON(b []byte) error { diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index 882893596a..bb7f149358 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -292,7 +292,7 @@ func TestBundleToTerraformPipelinePermissions(t *testing.T) { func TestBundleToTerraformModel(t *testing.T) { src := resources.MlflowModel{ - Model: ml.Model{ + CreateModelRequest: ml.CreateModelRequest{ Name: "name", Description: "description", Tags: []ml.ModelTag{ @@ -332,7 +332,7 @@ func TestBundleToTerraformModel(t *testing.T) { func TestBundleToTerraformModelPermissions(t *testing.T) { src := resources.MlflowModel{ - Model: ml.Model{ + CreateModelRequest: ml.CreateModelRequest{ Name: "name", }, Permissions: []resources.MlflowModelPermission{ @@ -766,7 +766,7 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, Models: map[string]*resources.MlflowModel{ "test_mlflow_model": { - Model: ml.Model{ + CreateModelRequest: ml.CreateModelRequest{ Name: "test_mlflow_model", }, }, @@ -910,12 +910,12 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, Models: map[string]*resources.MlflowModel{ "test_mlflow_model": { - Model: ml.Model{ + CreateModelRequest: ml.CreateModelRequest{ Name: "test_mlflow_model", }, }, "test_mlflow_model_new": { - Model: ml.Model{ + CreateModelRequest: ml.CreateModelRequest{ Name: "test_mlflow_model_new", }, }, diff --git a/bundle/deploy/terraform/interpolate_test.go b/bundle/deploy/terraform/interpolate_test.go index a9cc1dc80f..8df4c15d65 100644 --- a/bundle/deploy/terraform/interpolate_test.go +++ b/bundle/deploy/terraform/interpolate_test.go @@ -51,7 +51,7 @@ func TestInterpolate(t *testing.T) { }, Models: map[string]*resources.MlflowModel{ "my_model": { - Model: ml.Model{ + CreateModelRequest: ml.CreateModelRequest{ Name: "my_model", }, }, @@ -77,7 +77,7 @@ func TestInterpolate(t *testing.T) { assert.Equal(t, "${databricks_app.other_app.id}", j.Tags["other_app"]) m := b.Config.Resources.Models["my_model"] - assert.Equal(t, "my_model", m.Model.Name) + assert.Equal(t, "my_model", m.CreateModelRequest.Name) } func TestInterpolateUnknownResourceType(t *testing.T) { diff --git a/bundle/deploy/terraform/tfdyn/convert_model_test.go b/bundle/deploy/terraform/tfdyn/convert_model_test.go index 951965eaad..0b36034514 100644 --- a/bundle/deploy/terraform/tfdyn/convert_model_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_model_test.go @@ -15,7 +15,7 @@ import ( func TestConvertModel(t *testing.T) { src := resources.MlflowModel{ - Model: ml.Model{ + CreateModelRequest: ml.CreateModelRequest{ Name: "name", Description: "description", Tags: []ml.ModelTag{ diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index 375e7aecd9..7819672dc8 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -2733,65 +2733,6 @@ github.com/databricks/databricks-sdk-go/service/ml.ModelTag: "value": "description": |- The tag value. -github.com/databricks/databricks-sdk-go/service/ml.ModelVersion: - "creation_timestamp": - "description": |- - Timestamp recorded when this `model_version` was created. - "current_stage": - "description": |- - Current stage for this `model_version`. - "description": - "description": |- - Description of this `model_version`. - "last_updated_timestamp": - "description": |- - Timestamp recorded when metadata for this `model_version` was last updated. - "name": - "description": |- - Unique name of the model - "run_id": - "description": |- - MLflow run ID used when creating `model_version`, if `source` was generated by an - experiment run stored in MLflow tracking server. - "run_link": - "description": |- - Run Link: Direct link to the run that generated this version - "source": - "description": |- - URI indicating the location of the source model artifacts, used when creating `model_version` - "status": - "description": |- - Current status of `model_version` - "status_message": - "description": |- - Details on current `status`, if it is pending or failed. - "tags": - "description": |- - Tags: Additional metadata key-value pairs for this `model_version`. - "user_id": - "description": |- - User that created this `model_version`. - "version": - "description": |- - Model's version number. -github.com/databricks/databricks-sdk-go/service/ml.ModelVersionStatus: - "_": - "description": |- - Current status of `model_version` - "enum": - - |- - PENDING_REGISTRATION - - |- - FAILED_REGISTRATION - - |- - READY -github.com/databricks/databricks-sdk-go/service/ml.ModelVersionTag: - "key": - "description": |- - The tag key. - "value": - "description": |- - The tag value. github.com/databricks/databricks-sdk-go/service/pipelines.CronTrigger: "quartz_cron_schedule": {} "timezone_id": {} diff --git a/bundle/internal/schema/testdata/pass/ml.yml b/bundle/internal/schema/testdata/pass/ml.yml index b1558f101f..bf49e5fc79 100644 --- a/bundle/internal/schema/testdata/pass/ml.yml +++ b/bundle/internal/schema/testdata/pass/ml.yml @@ -26,12 +26,8 @@ variables: resources: models: mymodel: - creation_timestamp: 123 + name: mymodel description: "my model" - latest_versions: - - creation_timestamp: 123 - tags: ${var.complexvar.key1} - status: READY permissions: - service_principal_name: myserviceprincipal level: CAN_MANAGE diff --git a/bundle/permissions/workspace_root_test.go b/bundle/permissions/workspace_root_test.go index 417c6bde6f..3f0df91271 100644 --- a/bundle/permissions/workspace_root_test.go +++ b/bundle/permissions/workspace_root_test.go @@ -42,8 +42,8 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) { "pipeline_2": {CreatePipeline: pipelines.CreatePipeline{}}, }, Models: map[string]*resources.MlflowModel{ - "model_1": {Model: ml.Model{}}, - "model_2": {Model: ml.Model{}}, + "model_1": {}, + "model_2": {}, }, Experiments: map[string]*resources.MlflowExperiment{ "experiment_1": {Experiment: ml.Experiment{}}, @@ -102,8 +102,8 @@ func TestApplyWorkspaceRootPermissionsForAllPaths(t *testing.T) { "pipeline_2": {CreatePipeline: pipelines.CreatePipeline{}}, }, Models: map[string]*resources.MlflowModel{ - "model_1": {Model: ml.Model{}}, - "model_2": {Model: ml.Model{}}, + "model_1": {}, + "model_2": {}, }, Experiments: map[string]*resources.MlflowExperiment{ "experiment_1": {Experiment: ml.Experiment{}}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index a47dbdd173..ed7ab6bab3 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -770,22 +770,10 @@ { "type": "object", "properties": { - "creation_timestamp": { - "description": "Timestamp recorded when this `registered_model` was created.", - "$ref": "#/$defs/int64" - }, "description": { "description": "Description of this `registered_model`.", "$ref": "#/$defs/string" }, - "last_updated_timestamp": { - "description": "Timestamp recorded when metadata for this `registered_model` was last updated.", - "$ref": "#/$defs/int64" - }, - "latest_versions": { - "description": "Collection of latest model versions for each stage.\nOnly contains models with current `READY` status.", - "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ModelVersion" - }, "name": { "description": "Unique name for the model.", "$ref": "#/$defs/string" @@ -796,13 +784,12 @@ "tags": { "description": "Tags: Additional metadata key-value pairs for this `registered_model`.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ModelTag" - }, - "user_id": { - "description": "User that created this `registered_model`", - "$ref": "#/$defs/string" } }, "additionalProperties": false, + "required": [ + "name" + ], "markdownDescription": "The model resource allows you to define [legacy models](https://docs.databricks.com/api/workspace/modelregistry/createmodel) in bundles. Databricks recommends you use Unity Catalog [registered models](https://docs.databricks.com/dev-tools/bundles/reference.html#registered-model) instead." }, { @@ -5595,111 +5582,6 @@ } ] }, - "ml.ModelVersion": { - "oneOf": [ - { - "type": "object", - "properties": { - "creation_timestamp": { - "description": "Timestamp recorded when this `model_version` was created.", - "$ref": "#/$defs/int64" - }, - "current_stage": { - "description": "Current stage for this `model_version`.", - "$ref": "#/$defs/string" - }, - "description": { - "description": "Description of this `model_version`.", - "$ref": "#/$defs/string" - }, - "last_updated_timestamp": { - "description": "Timestamp recorded when metadata for this `model_version` was last updated.", - "$ref": "#/$defs/int64" - }, - "name": { - "description": "Unique name of the model", - "$ref": "#/$defs/string" - }, - "run_id": { - "description": "MLflow run ID used when creating `model_version`, if `source` was generated by an\nexperiment run stored in MLflow tracking server.", - "$ref": "#/$defs/string" - }, - "run_link": { - "description": "Run Link: Direct link to the run that generated this version", - "$ref": "#/$defs/string" - }, - "source": { - "description": "URI indicating the location of the source model artifacts, used when creating `model_version`", - "$ref": "#/$defs/string" - }, - "status": { - "description": "Current status of `model_version`", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelVersionStatus" - }, - "status_message": { - "description": "Details on current `status`, if it is pending or failed.", - "$ref": "#/$defs/string" - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs for this `model_version`.", - "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ModelVersionTag" - }, - "user_id": { - "description": "User that created this `model_version`.", - "$ref": "#/$defs/string" - }, - "version": { - "description": "Model's version number.", - "$ref": "#/$defs/string" - } - }, - "additionalProperties": false - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" - } - ] - }, - "ml.ModelVersionStatus": { - "oneOf": [ - { - "type": "string", - "description": "Current status of `model_version`", - "enum": [ - "PENDING_REGISTRATION", - "FAILED_REGISTRATION", - "READY" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" - } - ] - }, - "ml.ModelVersionTag": { - "oneOf": [ - { - "type": "object", - "properties": { - "key": { - "description": "The tag key.", - "$ref": "#/$defs/string" - }, - "value": { - "description": "The tag value.", - "$ref": "#/$defs/string" - } - }, - "additionalProperties": false - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" - } - ] - }, "pipelines.CronTrigger": { "oneOf": [ { @@ -8188,34 +8070,6 @@ } ] }, - "ml.ModelVersion": { - "oneOf": [ - { - "type": "array", - "items": { - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelVersion" - } - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" - } - ] - }, - "ml.ModelVersionTag": { - "oneOf": [ - { - "type": "array", - "items": { - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelVersionTag" - } - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" - } - ] - }, "pipelines.DayOfWeek": { "oneOf": [ { diff --git a/bundle/tests/run_as_test.go b/bundle/tests/run_as_test.go index 58174e44c3..e6d923c243 100644 --- a/bundle/tests/run_as_test.go +++ b/bundle/tests/run_as_test.go @@ -52,7 +52,7 @@ func TestRunAsForAllowed(t *testing.T) { assert.Equal(t, "", jobs["job_three"].RunAs.UserName) // Assert other resources are not affected. - assert.Equal(t, ml.Model{Name: "skynet"}, b.Config.Resources.Models["model_one"].Model) + assert.Equal(t, ml.CreateModelRequest{Name: "skynet"}, b.Config.Resources.Models["model_one"].CreateModelRequest) assert.Equal(t, catalog.CreateRegisteredModelRequest{Name: "skynet (in UC)"}, b.Config.Resources.RegisteredModels["model_two"].CreateRegisteredModelRequest) assert.Equal(t, ml.Experiment{Name: "experiment_one"}, b.Config.Resources.Experiments["experiment_one"].Experiment) } @@ -92,7 +92,7 @@ func TestRunAsForAllowedWithTargetOverride(t *testing.T) { assert.Equal(t, "", jobs["job_three"].RunAs.UserName) // Assert other resources are not affected. - assert.Equal(t, ml.Model{Name: "skynet"}, b.Config.Resources.Models["model_one"].Model) + assert.Equal(t, ml.CreateModelRequest{Name: "skynet"}, b.Config.Resources.Models["model_one"].CreateModelRequest) assert.Equal(t, catalog.CreateRegisteredModelRequest{Name: "skynet (in UC)"}, b.Config.Resources.RegisteredModels["model_two"].CreateRegisteredModelRequest) assert.Equal(t, ml.Experiment{Name: "experiment_one"}, b.Config.Resources.Experiments["experiment_one"].Experiment) } @@ -308,7 +308,7 @@ func TestLegacyRunAs(t *testing.T) { }, pipelines["nyc_taxi_pipeline"].Permissions[1]) // Assert other resources are not affected. - assert.Equal(t, ml.Model{Name: "skynet"}, b.Config.Resources.Models["model_one"].Model) + assert.Equal(t, ml.CreateModelRequest{Name: "skynet"}, b.Config.Resources.Models["model_one"].CreateModelRequest) assert.Equal(t, catalog.CreateRegisteredModelRequest{Name: "skynet (in UC)"}, b.Config.Resources.RegisteredModels["model_two"].CreateRegisteredModelRequest) assert.Equal(t, ml.Experiment{Name: "experiment_one"}, b.Config.Resources.Experiments["experiment_one"].Experiment) assert.Equal(t, serving.CreateServingEndpoint{Name: "skynet"}, b.Config.Resources.ModelServingEndpoints["model_serving_one"].CreateServingEndpoint)