From 221ddb53e9e17b285feab7713c21b0ff39770847 Mon Sep 17 00:00:00 2001 From: Domenico Corvasce Date: Wed, 20 Dec 2023 21:43:09 +0100 Subject: [PATCH 1/5] Add custom prefix option for development resources --- bundle/config/bundle.go | 4 ++++ bundle/config/mutator/process_target_mode.go | 3 +++ bundle/config/mutator/process_target_mode_test.go | 14 ++++++++++++++ bundle/config/root.go | 2 ++ bundle/config/target.go | 4 ++++ 5 files changed, 27 insertions(+) diff --git a/bundle/config/bundle.go b/bundle/config/bundle.go index 933e88bfad..d6c02555b6 100644 --- a/bundle/config/bundle.go +++ b/bundle/config/bundle.go @@ -41,6 +41,10 @@ type Bundle struct { // Annotated readonly as this should be set at the target level. Mode Mode `json:"mode,omitempty" bundle:"readonly"` + // Prefix for the names of the resources deployed in development mode + // The default one is "[dev ${workspace.current_user.userName}]" + ResourceNamePrefix string `json:"resource_name_prefix,omitempty" bundle:"readonly"` + // Overrides the compute used for jobs and other supported assets. ComputeID string `json:"compute_id,omitempty"` } diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 592e3612cf..3bce30c986 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -34,6 +34,9 @@ func transformDevelopmentMode(b *bundle.Bundle) error { shortName := b.Config.Workspace.CurrentUser.ShortName prefix := "[dev " + shortName + "] " + if b.Config.Bundle.ResourceNamePrefix != "" { + prefix = "[" + b.Config.Bundle.ResourceNamePrefix + "] " + } // Generate a normalized version of the short name that can be used as a tag value. tagValue := b.Tagging.NormalizeValue(shortName) diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index f02d788657..8ea5299f6d 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -146,6 +146,20 @@ func TestProcessTargetModeDevelopment(t *testing.T) { assert.Equal(t, "dev_lennart_registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) } +func TestProcessTargetModeDevelopmentCustomResourceNamePrefix(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Bundle.ResourceNamePrefix = "custom" + + m := ProcessTargetMode() + err := bundle.Apply(context.Background(), b, m) + require.NoError(t, err) + + // Job 1 + assert.Equal(t, "[custom] job1", b.Config.Resources.Jobs["job1"].Name) + assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["dev"], "lennart") + assert.Equal(t, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus, jobs.PauseStatusPaused) +} + func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { b := mockBundle(config.Development) b.Tagging = tags.ForCloud(&sdkconfig.Config{ diff --git a/bundle/config/root.go b/bundle/config/root.go index 94cc0b177b..f523edf9f4 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -227,6 +227,8 @@ func (r *Root) MergeTargetOverrides(target *Target) error { r.RunAs = target.RunAs } + r.Bundle.ResourceNamePrefix = target.ResourceNamePrefix + if target.Mode != "" { r.Bundle.Mode = target.Mode } diff --git a/bundle/config/target.go b/bundle/config/target.go index 158f256060..8612bc0c76 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -19,6 +19,10 @@ type Target struct { // development purposes. Mode Mode `json:"mode,omitempty"` + // Prefix for the names of the resources deployed in development mode + // The default one is "[dev ${workspace.current_user.userName}]" + ResourceNamePrefix string `json:"resource_name_prefix,omitempty"` + // Overrides the compute used for jobs and other supported assets. ComputeID string `json:"compute_id,omitempty"` From 1f252b56dde219cfcdba34f3b829c9f52b0fb901 Mon Sep 17 00:00:00 2001 From: Domenico Corvasce Date: Wed, 20 Dec 2023 21:54:38 +0100 Subject: [PATCH 2/5] Fix docstrings --- bundle/config/bundle.go | 2 +- bundle/config/mutator/process_target_mode_test.go | 14 ++++++++++++++ bundle/config/target.go | 4 ++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/bundle/config/bundle.go b/bundle/config/bundle.go index d6c02555b6..1735e7bc1d 100644 --- a/bundle/config/bundle.go +++ b/bundle/config/bundle.go @@ -42,7 +42,7 @@ type Bundle struct { Mode Mode `json:"mode,omitempty" bundle:"readonly"` // Prefix for the names of the resources deployed in development mode - // The default one is "[dev ${workspace.current_user.userName}]" + // The default one is "dev ${workspace.current_user.shortName}" ResourceNamePrefix string `json:"resource_name_prefix,omitempty" bundle:"readonly"` // Overrides the compute used for jobs and other supported assets. diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 8ea5299f6d..c05d6f769e 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -160,6 +160,20 @@ func TestProcessTargetModeDevelopmentCustomResourceNamePrefix(t *testing.T) { assert.Equal(t, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus, jobs.PauseStatusPaused) } +func TestProcessTargetModeDevelopmentCustomDeploymentTag(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Bundle.ResourceNamePrefix = "custom" + + m := ProcessTargetMode() + err := bundle.Apply(context.Background(), b, m) + require.NoError(t, err) + + // Job 1 + assert.Equal(t, "[custom] job1", b.Config.Resources.Jobs["job1"].Name) + assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["dev"], "lennart") + assert.Equal(t, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus, jobs.PauseStatusPaused) +} + func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { b := mockBundle(config.Development) b.Tagging = tags.ForCloud(&sdkconfig.Config{ diff --git a/bundle/config/target.go b/bundle/config/target.go index 8612bc0c76..f8e9075631 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -20,7 +20,7 @@ type Target struct { Mode Mode `json:"mode,omitempty"` // Prefix for the names of the resources deployed in development mode - // The default one is "[dev ${workspace.current_user.userName}]" + // The default one is "dev ${workspace.current_user.shortName}" ResourceNamePrefix string `json:"resource_name_prefix,omitempty"` // Overrides the compute used for jobs and other supported assets. @@ -50,7 +50,7 @@ type Target struct { const ( // Development mode: deployments done purely for running things in development. - // Any deployed resources will be marked as "dev" and might be hidden or cleaned up. + // Any deployed resources will be marked as "dev" (or custom a `tag`) and might be hidden or cleaned up. Development Mode = "development" // Production mode: deployments done for production purposes. From ead50928c6656c726de6abddee97fa7c4523829e Mon Sep 17 00:00:00 2001 From: Domenico Corvasce Date: Wed, 20 Dec 2023 22:14:20 +0100 Subject: [PATCH 3/5] Extend custom prefix to model endpoints --- bundle/config/mutator/process_target_mode.go | 15 +++++++--- .../mutator/process_target_mode_test.go | 29 +++++++++++++++---- bundle/config/root.go | 4 ++- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 3bce30c986..158b5379dd 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -26,6 +26,12 @@ func (m *processTargetMode) Name() string { return "ProcessTargetMode" } +// Normalize the resource name prefix in order to use it +// in constrained contexts such as model serving endpoints names +func normalizeResourceNamePrefix(prefix string) string { + return strings.Map(replaceNonAlphanumeric, strings.ToLower(strings.ReplaceAll(prefix, " ", "_") + "_")) +} + // Mark all resources as being for 'development' purposes, i.e. // changing their their name, adding tags, and (in the future) // marking them as 'hidden' in the UI. @@ -34,8 +40,11 @@ func transformDevelopmentMode(b *bundle.Bundle) error { shortName := b.Config.Workspace.CurrentUser.ShortName prefix := "[dev " + shortName + "] " + normalizedPrefix := "dev_" + shortName + "_" + if b.Config.Bundle.ResourceNamePrefix != "" { prefix = "[" + b.Config.Bundle.ResourceNamePrefix + "] " + normalizedPrefix = normalizeResourceNamePrefix(b.Config.Bundle.ResourceNamePrefix) } // Generate a normalized version of the short name that can be used as a tag value. @@ -89,14 +98,12 @@ func transformDevelopmentMode(b *bundle.Bundle) error { } for i := range r.ModelServingEndpoints { - prefix = "dev_" + b.Config.Workspace.CurrentUser.ShortName + "_" - r.ModelServingEndpoints[i].Name = prefix + r.ModelServingEndpoints[i].Name + r.ModelServingEndpoints[i].Name = normalizedPrefix + r.ModelServingEndpoints[i].Name // (model serving doesn't yet support tags) } for i := range r.RegisteredModels { - prefix = "dev_" + b.Config.Workspace.CurrentUser.ShortName + "_" - r.RegisteredModels[i].Name = prefix + r.RegisteredModels[i].Name + r.RegisteredModels[i].Name = normalizedPrefix + r.RegisteredModels[i].Name // (registered models in Unity Catalog don't yet support tags) } diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index c05d6f769e..af2f48434b 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -148,16 +148,35 @@ func TestProcessTargetModeDevelopment(t *testing.T) { func TestProcessTargetModeDevelopmentCustomResourceNamePrefix(t *testing.T) { b := mockBundle(config.Development) - b.Config.Bundle.ResourceNamePrefix = "custom" + b.Config.Bundle.ResourceNamePrefix = "Staging @us-east-1" m := ProcessTargetMode() err := bundle.Apply(context.Background(), b, m) require.NoError(t, err) // Job 1 - assert.Equal(t, "[custom] job1", b.Config.Resources.Jobs["job1"].Name) - assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["dev"], "lennart") - assert.Equal(t, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus, jobs.PauseStatusPaused) + assert.Equal(t, "[Staging @us-east-1] job1", b.Config.Resources.Jobs["job1"].Name) + + // Job 2 + assert.Equal(t, "[Staging @us-east-1] job2", b.Config.Resources.Jobs["job2"].Name) + + // Pipeline 1 + assert.Equal(t, "[Staging @us-east-1] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name) + + // Experiment 1 + assert.Equal(t, "/Users/lennart.kats@databricks.com/[Staging @us-east-1] experiment1", b.Config.Resources.Experiments["experiment1"].Name) + + // Experiment 2 + assert.Equal(t, "[Staging @us-east-1] experiment2", b.Config.Resources.Experiments["experiment2"].Name) + + // Model 1 + assert.Equal(t, "[Staging @us-east-1] model1", b.Config.Resources.Models["model1"].Name) + + // Model serving endpoint 1 + assert.Equal(t, "staging__us_east_1_servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) + + // Registered model 1 + assert.Equal(t, "staging__us_east_1_registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) } func TestProcessTargetModeDevelopmentCustomDeploymentTag(t *testing.T) { @@ -170,8 +189,6 @@ func TestProcessTargetModeDevelopmentCustomDeploymentTag(t *testing.T) { // Job 1 assert.Equal(t, "[custom] job1", b.Config.Resources.Jobs["job1"].Name) - assert.Equal(t, b.Config.Resources.Jobs["job1"].Tags["dev"], "lennart") - assert.Equal(t, b.Config.Resources.Jobs["job1"].Schedule.PauseStatus, jobs.PauseStatusPaused) } func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { diff --git a/bundle/config/root.go b/bundle/config/root.go index f523edf9f4..3221859212 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -227,7 +227,9 @@ func (r *Root) MergeTargetOverrides(target *Target) error { r.RunAs = target.RunAs } - r.Bundle.ResourceNamePrefix = target.ResourceNamePrefix + if target.ResourceNamePrefix != "" { + r.Bundle.ResourceNamePrefix = target.ResourceNamePrefix + } if target.Mode != "" { r.Bundle.Mode = target.Mode From 643c1dd1e5db9cca33fa4f866aabea271ad9176d Mon Sep 17 00:00:00 2001 From: Domenico Corvasce Date: Wed, 20 Dec 2023 22:32:52 +0100 Subject: [PATCH 4/5] Format code --- bundle/config/mutator/process_target_mode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 158b5379dd..8f6f8d6888 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -29,7 +29,7 @@ func (m *processTargetMode) Name() string { // Normalize the resource name prefix in order to use it // in constrained contexts such as model serving endpoints names func normalizeResourceNamePrefix(prefix string) string { - return strings.Map(replaceNonAlphanumeric, strings.ToLower(strings.ReplaceAll(prefix, " ", "_") + "_")) + return strings.Map(replaceNonAlphanumeric, strings.ToLower(strings.ReplaceAll(prefix, " ", "_")+"_")) } // Mark all resources as being for 'development' purposes, i.e. From 688af35ce60b16bded8e53bb91b24aab0878cf1a Mon Sep 17 00:00:00 2001 From: Domenico Corvasce Date: Wed, 20 Dec 2023 23:11:11 +0100 Subject: [PATCH 5/5] Remove changes reversed to a future pull request --- bundle/config/mutator/process_target_mode_test.go | 12 ------------ bundle/config/target.go | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index af2f48434b..bb3c38af5b 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -179,18 +179,6 @@ func TestProcessTargetModeDevelopmentCustomResourceNamePrefix(t *testing.T) { assert.Equal(t, "staging__us_east_1_registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) } -func TestProcessTargetModeDevelopmentCustomDeploymentTag(t *testing.T) { - b := mockBundle(config.Development) - b.Config.Bundle.ResourceNamePrefix = "custom" - - m := ProcessTargetMode() - err := bundle.Apply(context.Background(), b, m) - require.NoError(t, err) - - // Job 1 - assert.Equal(t, "[custom] job1", b.Config.Resources.Jobs["job1"].Name) -} - func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { b := mockBundle(config.Development) b.Tagging = tags.ForCloud(&sdkconfig.Config{ diff --git a/bundle/config/target.go b/bundle/config/target.go index f8e9075631..7366aa6d5f 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -50,7 +50,7 @@ type Target struct { const ( // Development mode: deployments done purely for running things in development. - // Any deployed resources will be marked as "dev" (or custom a `tag`) and might be hidden or cleaned up. + // Any deployed resources will be marked as "dev" and might be hidden or cleaned up. Development Mode = "development" // Production mode: deployments done for production purposes.