From 5a0cc8098ba4c4640c8d2bddc683b58f1aa5b17c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 16 Oct 2023 14:46:41 +0200 Subject: [PATCH 01/11] Allow referencing bundle resources by name --- .codegen.json | 3 +- .codegen/resolvers.go.tmpl | 37 +++ .gitattributes | 1 + .../mutator/resolve_resource_references.go | 51 ++++ .../resolve_resource_references_test.go | 182 +++++++++++++ bundle/config/variable/variable.go | 4 + bundle/phases/initialize.go | 1 + bundle/resolvers/resolvers.go | 253 ++++++++++++++++++ 8 files changed, 531 insertions(+), 1 deletion(-) create mode 100644 .codegen/resolvers.go.tmpl create mode 100644 bundle/config/mutator/resolve_resource_references.go create mode 100644 bundle/config/mutator/resolve_resource_references_test.go create mode 100755 bundle/resolvers/resolvers.go diff --git a/.codegen.json b/.codegen.json index da4f3dd610..9d6e6fda6d 100644 --- a/.codegen.json +++ b/.codegen.json @@ -5,7 +5,8 @@ }, "batch": { ".codegen/cmds-workspace.go.tmpl": "cmd/workspace/cmd.go", - ".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go" + ".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go", + ".codegen/resolvers.go.tmpl": "bundle/resolvers/resolvers.go" }, "toolchain": { "required": ["go"] diff --git a/.codegen/resolvers.go.tmpl b/.codegen/resolvers.go.tmpl new file mode 100644 index 0000000000..bbf516980f --- /dev/null +++ b/.codegen/resolvers.go.tmpl @@ -0,0 +1,37 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package resolvers + +import ( + "context" + "fmt" + "github.com/databricks/cli/bundle" +) + +type ResolverFunc func(ctx context.Context, b *bundle.Bundle, name string) (string, error) + +func Resolvers() map[string](ResolverFunc) { + resolvers := make(map[string](ResolverFunc), 0) + {{range .Services -}} + {{- if not .IsAccounts -}} + {{- if and .List .List.GetByName }} + resolvers["{{.KebabName}}"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.{{.PascalName}}.GetBy{{range .List.NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity{{ template "field-path" .List.NamedIdMap.IdPath }}), nil + } + {{- end -}} + {{- end -}} + {{- end}} + + return resolvers +} + + +{{- define "field-path" -}} + {{- range .}}.{{.PascalName}}{{end}} +{{- end -}} diff --git a/.gitattributes b/.gitattributes index f50218feda..fc2bc671d3 100755 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ +bundle/resolvers/resolvers.go linguist-generated=true cmd/account/access-control/access-control.go linguist-generated=true cmd/account/billable-usage/billable-usage.go linguist-generated=true cmd/account/budgets/budgets.go linguist-generated=true diff --git a/bundle/config/mutator/resolve_resource_references.go b/bundle/config/mutator/resolve_resource_references.go new file mode 100644 index 0000000000..520123711f --- /dev/null +++ b/bundle/config/mutator/resolve_resource_references.go @@ -0,0 +1,51 @@ +package mutator + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/resolvers" +) + +var separator string = ":" + +type resolveResourceReferences struct { + resolvers map[string](resolvers.ResolverFunc) +} + +func ResolveResourceReferences() bundle.Mutator { + return &resolveResourceReferences{ + resolvers: resolvers.Resolvers(), + } +} + +func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) error { + for _, v := range b.Config.Variables { + value := v.Value + parts := strings.Split(*value, separator) + if len(parts) != 2 { + continue + } + + resource, name := parts[0], parts[1] + resolver, ok := m.resolvers[resource] + if !ok { + return fmt.Errorf("unable to resolve resource reference %s, no resovler for %s", *value, resource) + } + + id, err := resolver(ctx, b, name) + if err != nil { + return fmt.Errorf("failed to resolve resource reference %s, err: %w", *value, err) + } + + v.Replace(id) + } + + return nil +} + +func (*resolveResourceReferences) Name() string { + return "ResolveResourceReferences" +} diff --git a/bundle/config/mutator/resolve_resource_references_test.go b/bundle/config/mutator/resolve_resource_references_test.go new file mode 100644 index 0000000000..82024faf09 --- /dev/null +++ b/bundle/config/mutator/resolve_resource_references_test.go @@ -0,0 +1,182 @@ +package mutator + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/variable" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/stretchr/testify/require" +) + +type MockClusterService struct{} + +// ChangeOwner implements compute.ClustersService. +func (MockClusterService) ChangeOwner(ctx context.Context, request compute.ChangeClusterOwner) error { + panic("unimplemented") +} + +// Create implements compute.ClustersService. +func (MockClusterService) Create(ctx context.Context, request compute.CreateCluster) (*compute.CreateClusterResponse, error) { + panic("unimplemented") +} + +// Delete implements compute.ClustersService. +func (MockClusterService) Delete(ctx context.Context, request compute.DeleteCluster) error { + panic("unimplemented") +} + +// Edit implements compute.ClustersService. +func (MockClusterService) Edit(ctx context.Context, request compute.EditCluster) error { + panic("unimplemented") +} + +// Events implements compute.ClustersService. +func (MockClusterService) Events(ctx context.Context, request compute.GetEvents) (*compute.GetEventsResponse, error) { + panic("unimplemented") +} + +// Get implements compute.ClustersService. +func (MockClusterService) Get(ctx context.Context, request compute.GetClusterRequest) (*compute.ClusterDetails, error) { + panic("unimplemented") +} + +// GetPermissionLevels implements compute.ClustersService. +func (MockClusterService) GetPermissionLevels(ctx context.Context, request compute.GetClusterPermissionLevelsRequest) (*compute.GetClusterPermissionLevelsResponse, error) { + panic("unimplemented") +} + +// GetPermissions implements compute.ClustersService. +func (MockClusterService) GetPermissions(ctx context.Context, request compute.GetClusterPermissionsRequest) (*compute.ClusterPermissions, error) { + panic("unimplemented") +} + +// List implements compute.ClustersService. +func (MockClusterService) List(ctx context.Context, request compute.ListClustersRequest) (*compute.ListClustersResponse, error) { + return &compute.ListClustersResponse{ + Clusters: []compute.ClusterDetails{ + {ClusterId: "1234-5678-abcd", ClusterName: "Some Custom Cluster"}, + {ClusterId: "9876-5432-xywz", ClusterName: "Some Other Name"}, + }, + }, nil +} + +// ListNodeTypes implements compute.ClustersService. +func (MockClusterService) ListNodeTypes(ctx context.Context) (*compute.ListNodeTypesResponse, error) { + panic("unimplemented") +} + +// ListZones implements compute.ClustersService. +func (MockClusterService) ListZones(ctx context.Context) (*compute.ListAvailableZonesResponse, error) { + panic("unimplemented") +} + +// PermanentDelete implements compute.ClustersService. +func (MockClusterService) PermanentDelete(ctx context.Context, request compute.PermanentDeleteCluster) error { + panic("unimplemented") +} + +// Pin implements compute.ClustersService. +func (MockClusterService) Pin(ctx context.Context, request compute.PinCluster) error { + panic("unimplemented") +} + +// Resize implements compute.ClustersService. +func (MockClusterService) Resize(ctx context.Context, request compute.ResizeCluster) error { + panic("unimplemented") +} + +// Restart implements compute.ClustersService. +func (MockClusterService) Restart(ctx context.Context, request compute.RestartCluster) error { + panic("unimplemented") +} + +// SetPermissions implements compute.ClustersService. +func (MockClusterService) SetPermissions(ctx context.Context, request compute.ClusterPermissionsRequest) (*compute.ClusterPermissions, error) { + panic("unimplemented") +} + +// SparkVersions implements compute.ClustersService. +func (MockClusterService) SparkVersions(ctx context.Context) (*compute.GetSparkVersionsResponse, error) { + panic("unimplemented") +} + +// Start implements compute.ClustersService. +func (MockClusterService) Start(ctx context.Context, request compute.StartCluster) error { + panic("unimplemented") +} + +// Unpin implements compute.ClustersService. +func (MockClusterService) Unpin(ctx context.Context, request compute.UnpinCluster) error { + panic("unimplemented") +} + +// UpdatePermissions implements compute.ClustersService. +func (MockClusterService) UpdatePermissions(ctx context.Context, request compute.ClusterPermissionsRequest) (*compute.ClusterPermissions, error) { + panic("unimplemented") +} + +func TestResolveClusterReference(t *testing.T) { + clusterRef := "clusters:Some Custom Cluster" + justString := "random string" + b := &bundle.Bundle{ + Config: config.Root{ + Variables: map[string]*variable.Variable{ + "my-cluster-id": { + Value: &clusterRef, + }, + "some-variable": { + Value: &justString, + }, + }, + }, + } + + b.WorkspaceClient().Clusters.WithImpl(MockClusterService{}) + + err := bundle.Apply(context.Background(), b, ResolveResourceReferences()) + require.NoError(t, err) + require.Equal(t, "1234-5678-abcd", *b.Config.Variables["my-cluster-id"].Value) +} + +func TestResolveNonExistentClusterReference(t *testing.T) { + clusterRef := "clusters:Random" + justString := "random string" + b := &bundle.Bundle{ + Config: config.Root{ + Variables: map[string]*variable.Variable{ + "my-cluster-id": { + Value: &clusterRef, + }, + "some-variable": { + Value: &justString, + }, + }, + }, + } + + b.WorkspaceClient().Clusters.WithImpl(MockClusterService{}) + + err := bundle.Apply(context.Background(), b, ResolveResourceReferences()) + require.ErrorContains(t, err, "failed to resolve resource reference clusters:Random, err: ClusterDetails named 'Random' does not exist") +} + +func TestResolveNonExistentResourceType(t *testing.T) { + clusterRef := "donotexist:Random" + b := &bundle.Bundle{ + Config: config.Root{ + Variables: map[string]*variable.Variable{ + "my-cluster-id": { + Value: &clusterRef, + }, + }, + }, + } + + b.WorkspaceClient().Clusters.WithImpl(MockClusterService{}) + + err := bundle.Apply(context.Background(), b, ResolveResourceReferences()) + require.ErrorContains(t, err, "unable to resolve resource reference donotexist:Random, no resovler for donotexist") +} diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index 73925d432e..10ce3e0f90 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -44,3 +44,7 @@ func (v *Variable) Set(val string) error { v.Value = &val return nil } + +func (v *Variable) Replace(val string) { + v.Value = &val +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index e03a63364c..0d90f91b42 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -24,6 +24,7 @@ func Initialize() bundle.Mutator { mutator.ExpandWorkspaceRoot(), mutator.DefineDefaultWorkspacePaths(), mutator.SetVariables(), + mutator.ResolveResourceReferences(), interpolation.Interpolate( interpolation.IncludeLookupsInPath("bundle"), interpolation.IncludeLookupsInPath("workspace"), diff --git a/bundle/resolvers/resolvers.go b/bundle/resolvers/resolvers.go new file mode 100755 index 0000000000..f253dd79a9 --- /dev/null +++ b/bundle/resolvers/resolvers.go @@ -0,0 +1,253 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package resolvers + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" +) + +type ResolverFunc func(ctx context.Context, b *bundle.Bundle, name string) (string, error) + +func Resolvers() map[string](ResolverFunc) { + resolvers := make(map[string](ResolverFunc), 0) + + resolvers["alerts"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Alerts.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["cluster-policies"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.ClusterPolicies.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.PolicyId), nil + } + resolvers["clusters"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Clusters.GetByClusterName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.ClusterId), nil + } + resolvers["connections"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Connections.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.FullName), nil + } + resolvers["dashboards"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Dashboards.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["data-sources"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.DataSources.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["git-credentials"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.GitCredentials.GetByGitProvider(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.CredentialId), nil + } + resolvers["global-init-scripts"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.GlobalInitScripts.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.ScriptId), nil + } + resolvers["groups"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Groups.GetByDisplayName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["instance-pools"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.InstancePools.GetByInstancePoolName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.InstancePoolId), nil + } + resolvers["ip-access-lists"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.IpAccessLists.GetByLabel(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.ListId), nil + } + resolvers["jobs"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Jobs.GetBySettingsName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.JobId), nil + } + resolvers["metastores"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Metastores.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.MetastoreId), nil + } + resolvers["pipelines"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Pipelines.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.PipelineId), nil + } + resolvers["queries"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Queries.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["registered-models"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.RegisteredModels.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.FullName), nil + } + resolvers["repos"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Repos.GetByPath(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["schemas"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Schemas.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.FullName), nil + } + resolvers["service-principals"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.ServicePrincipals.GetByDisplayName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["tables"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Tables.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.TableId), nil + } + resolvers["token-management"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.TokenManagement.GetByComment(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.TokenId), nil + } + resolvers["tokens"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Tokens.GetByComment(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.TokenId), nil + } + resolvers["users"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Users.GetByUserName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["volumes"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Volumes.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.VolumeId), nil + } + resolvers["warehouses"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Warehouses.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["workspace"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.Workspace.GetByPath(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.ObjectId), nil + } + + return resolvers +} From 33578a66b690669c1d86644c0064c541d1b88828 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 19 Oct 2023 16:51:31 +0200 Subject: [PATCH 02/11] use lookup syntax --- .../mutator/resolve_resource_references.go | 24 ++++++++++++----- .../resolve_resource_references_test.go | 26 ++++++++++++++++--- bundle/config/mutator/set_variables.go | 6 +++++ bundle/config/variable/variable.go | 10 ++++--- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/bundle/config/mutator/resolve_resource_references.go b/bundle/config/mutator/resolve_resource_references.go index 520123711f..43d0059120 100644 --- a/bundle/config/mutator/resolve_resource_references.go +++ b/bundle/config/mutator/resolve_resource_references.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/resolvers" + "github.com/databricks/cli/libs/log" ) var separator string = ":" @@ -22,25 +23,34 @@ func ResolveResourceReferences() bundle.Mutator { } func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) error { - for _, v := range b.Config.Variables { - value := v.Value - parts := strings.Split(*value, separator) - if len(parts) != 2 { + for k, v := range b.Config.Variables { + if v.Lookup == "" { + continue + } + + if v.HasValue() { + log.Debugf(ctx, "Ignoring '%s' lookup for the variable '%s' because the value is set", v.Lookup, k) continue } + lookup := v.Lookup + parts := strings.Split(lookup, separator) + if len(parts) != 2 { + return fmt.Errorf("incorrect lookup specified %s", lookup) + } + resource, name := parts[0], parts[1] resolver, ok := m.resolvers[resource] if !ok { - return fmt.Errorf("unable to resolve resource reference %s, no resovler for %s", *value, resource) + return fmt.Errorf("unable to resolve resource reference %s, no resovler for %s", lookup, resource) } id, err := resolver(ctx, b, name) if err != nil { - return fmt.Errorf("failed to resolve resource reference %s, err: %w", *value, err) + return fmt.Errorf("failed to resolve resource reference %s, err: %w", lookup, err) } - v.Replace(id) + v.Set(id) } return nil diff --git a/bundle/config/mutator/resolve_resource_references_test.go b/bundle/config/mutator/resolve_resource_references_test.go index 82024faf09..8522f58cc8 100644 --- a/bundle/config/mutator/resolve_resource_references_test.go +++ b/bundle/config/mutator/resolve_resource_references_test.go @@ -125,7 +125,7 @@ func TestResolveClusterReference(t *testing.T) { Config: config.Root{ Variables: map[string]*variable.Variable{ "my-cluster-id": { - Value: &clusterRef, + Lookup: clusterRef, }, "some-variable": { Value: &justString, @@ -148,7 +148,7 @@ func TestResolveNonExistentClusterReference(t *testing.T) { Config: config.Root{ Variables: map[string]*variable.Variable{ "my-cluster-id": { - Value: &clusterRef, + Lookup: clusterRef, }, "some-variable": { Value: &justString, @@ -169,7 +169,7 @@ func TestResolveNonExistentResourceType(t *testing.T) { Config: config.Root{ Variables: map[string]*variable.Variable{ "my-cluster-id": { - Value: &clusterRef, + Lookup: clusterRef, }, }, }, @@ -180,3 +180,23 @@ func TestResolveNonExistentResourceType(t *testing.T) { err := bundle.Apply(context.Background(), b, ResolveResourceReferences()) require.ErrorContains(t, err, "unable to resolve resource reference donotexist:Random, no resovler for donotexist") } + +func TestNoLookupIfVariableIsSet(t *testing.T) { + clusterRef := "donotexist:Random" + b := &bundle.Bundle{ + Config: config.Root{ + Variables: map[string]*variable.Variable{ + "my-cluster-id": { + Lookup: clusterRef, + }, + }, + }, + } + + b.WorkspaceClient().Clusters.WithImpl(MockClusterService{}) + b.Config.Variables["my-cluster-id"].Set("random value") + + err := bundle.Apply(context.Background(), b, ResolveResourceReferences()) + require.NoError(t, err) + require.Equal(t, "random value", *b.Config.Variables["my-cluster-id"].Value) +} diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index 4bf8ff82a0..c59101479c 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -46,6 +46,12 @@ func setVariable(ctx context.Context, v *variable.Variable, name string) error { return nil } + // case: Defined a variable for named lookup for a resource + // It will be resolved later in ResolveResourceReferences mutator + if v.Lookup != "" { + return nil + } + // We should have had a value to set for the variable at this point. // TODO: use cmdio to request values for unassigned variables if current // terminal is a tty. Tracked in https://github.com/databricks/cli/issues/379 diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index 10ce3e0f90..f2a2dd8e8b 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -24,6 +24,12 @@ type Variable struct { // 5. Throw error, since if no default value is defined, then the variable // is required Value *string `json:"value,omitempty" bundle:"readonly"` + + // A string value that represents a reference to the remote resource by name + // Format: ":" + // The value of this field will be used to lookup the resource by name + // And assign the value of the variable to ID of the resource found. + Lookup string `json:"lookup,omitempty"` } // True if the variable has been assigned a default value. Variables without a @@ -44,7 +50,3 @@ func (v *Variable) Set(val string) error { v.Value = &val return nil } - -func (v *Variable) Replace(val string) { - v.Value = &val -} From 6969f841324ea876428cf0c54a6802e1aee97939 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 20 Oct 2023 17:46:51 +0200 Subject: [PATCH 03/11] parallelised + fixes --- .codegen/resolvers.go.tmpl | 17 +++ .../mutator/resolve_resource_references.go | 32 ++-- .../resolve_resource_references_test.go | 17 ++- bundle/resolvers/resolvers.go | 144 ------------------ 4 files changed, 47 insertions(+), 163 deletions(-) diff --git a/.codegen/resolvers.go.tmpl b/.codegen/resolvers.go.tmpl index bbf516980f..da8cd72ac9 100644 --- a/.codegen/resolvers.go.tmpl +++ b/.codegen/resolvers.go.tmpl @@ -2,6 +2,21 @@ package resolvers +{{ $allowlist := + list + "alerts" + "clusters" + "cluster-policies" + "clusters" + "dashboards" + "instance-pools" + "jobs" + "metastores" + "pipelines" + "queries" + "warehouses" +}} + import ( "context" "fmt" @@ -13,6 +28,7 @@ type ResolverFunc func(ctx context.Context, b *bundle.Bundle, name string) (stri func Resolvers() map[string](ResolverFunc) { resolvers := make(map[string](ResolverFunc), 0) {{range .Services -}} + {{- if in $allowlist .KebabName -}} {{- if not .IsAccounts -}} {{- if and .List .List.GetByName }} resolvers["{{.KebabName}}"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { @@ -26,6 +42,7 @@ func Resolvers() map[string](ResolverFunc) { } {{- end -}} {{- end -}} + {{- end -}} {{- end}} return resolvers diff --git a/bundle/config/mutator/resolve_resource_references.go b/bundle/config/mutator/resolve_resource_references.go index 43d0059120..f2e665a4bf 100644 --- a/bundle/config/mutator/resolve_resource_references.go +++ b/bundle/config/mutator/resolve_resource_references.go @@ -8,12 +8,13 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/resolvers" "github.com/databricks/cli/libs/log" + "golang.org/x/sync/errgroup" ) -var separator string = ":" +const separator string = ":" type resolveResourceReferences struct { - resolvers map[string](resolvers.ResolverFunc) + resolvers map[string]resolvers.ResolverFunc } func ResolveResourceReferences() bundle.Mutator { @@ -23,7 +24,10 @@ func ResolveResourceReferences() bundle.Mutator { } func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) error { - for k, v := range b.Config.Variables { + errs, errCtx := errgroup.WithContext(ctx) + + for k := range b.Config.Variables { + v := b.Config.Variables[k] if v.Lookup == "" { continue } @@ -34,26 +38,28 @@ func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) } lookup := v.Lookup - parts := strings.Split(lookup, separator) - if len(parts) != 2 { + resource, name, ok := strings.Cut(lookup, separator) + if !ok { return fmt.Errorf("incorrect lookup specified %s", lookup) } - resource, name := parts[0], parts[1] resolver, ok := m.resolvers[resource] if !ok { - return fmt.Errorf("unable to resolve resource reference %s, no resovler for %s", lookup, resource) + return fmt.Errorf("unable to resolve resource reference %s, no resolvers for %s", lookup, resource) } - id, err := resolver(ctx, b, name) - if err != nil { - return fmt.Errorf("failed to resolve resource reference %s, err: %w", lookup, err) - } + errs.Go(func() error { + id, err := resolver(errCtx, b, name) + if err != nil { + return fmt.Errorf("failed to resolve %s reference %s, err: %w", resource, lookup, err) + } - v.Set(id) + v.Set(id) + return nil + }) } - return nil + return errs.Wait() } func (*resolveResourceReferences) Name() string { diff --git a/bundle/config/mutator/resolve_resource_references_test.go b/bundle/config/mutator/resolve_resource_references_test.go index 8522f58cc8..2b6793a0c0 100644 --- a/bundle/config/mutator/resolve_resource_references_test.go +++ b/bundle/config/mutator/resolve_resource_references_test.go @@ -119,13 +119,17 @@ func (MockClusterService) UpdatePermissions(ctx context.Context, request compute } func TestResolveClusterReference(t *testing.T) { - clusterRef := "clusters:Some Custom Cluster" + clusterRef1 := "clusters:Some Custom Cluster" + clusterRef2 := "clusters:Some Other Name" justString := "random string" b := &bundle.Bundle{ Config: config.Root{ Variables: map[string]*variable.Variable{ - "my-cluster-id": { - Lookup: clusterRef, + "my-cluster-id-1": { + Lookup: clusterRef1, + }, + "my-cluster-id-2": { + Lookup: clusterRef2, }, "some-variable": { Value: &justString, @@ -138,7 +142,8 @@ func TestResolveClusterReference(t *testing.T) { err := bundle.Apply(context.Background(), b, ResolveResourceReferences()) require.NoError(t, err) - require.Equal(t, "1234-5678-abcd", *b.Config.Variables["my-cluster-id"].Value) + require.Equal(t, "1234-5678-abcd", *b.Config.Variables["my-cluster-id-1"].Value) + require.Equal(t, "9876-5432-xywz", *b.Config.Variables["my-cluster-id-2"].Value) } func TestResolveNonExistentClusterReference(t *testing.T) { @@ -160,7 +165,7 @@ func TestResolveNonExistentClusterReference(t *testing.T) { b.WorkspaceClient().Clusters.WithImpl(MockClusterService{}) err := bundle.Apply(context.Background(), b, ResolveResourceReferences()) - require.ErrorContains(t, err, "failed to resolve resource reference clusters:Random, err: ClusterDetails named 'Random' does not exist") + require.ErrorContains(t, err, "failed to resolve clusters reference clusters:Random, err: ClusterDetails named 'Random' does not exist") } func TestResolveNonExistentResourceType(t *testing.T) { @@ -178,7 +183,7 @@ func TestResolveNonExistentResourceType(t *testing.T) { b.WorkspaceClient().Clusters.WithImpl(MockClusterService{}) err := bundle.Apply(context.Background(), b, ResolveResourceReferences()) - require.ErrorContains(t, err, "unable to resolve resource reference donotexist:Random, no resovler for donotexist") + require.ErrorContains(t, err, "unable to resolve resource reference donotexist:Random, no resolvers for donotexist") } func TestNoLookupIfVariableIsSet(t *testing.T) { diff --git a/bundle/resolvers/resolvers.go b/bundle/resolvers/resolvers.go index f253dd79a9..14a5a69966 100755 --- a/bundle/resolvers/resolvers.go +++ b/bundle/resolvers/resolvers.go @@ -41,15 +41,6 @@ func Resolvers() map[string](ResolverFunc) { return fmt.Sprint(entity.ClusterId), nil } - resolvers["connections"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Connections.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.FullName), nil - } resolvers["dashboards"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { w := b.WorkspaceClient() entity, err := w.Dashboards.GetByName(ctx, name) @@ -59,42 +50,6 @@ func Resolvers() map[string](ResolverFunc) { return fmt.Sprint(entity.Id), nil } - resolvers["data-sources"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.DataSources.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.Id), nil - } - resolvers["git-credentials"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.GitCredentials.GetByGitProvider(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.CredentialId), nil - } - resolvers["global-init-scripts"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.GlobalInitScripts.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.ScriptId), nil - } - resolvers["groups"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Groups.GetByDisplayName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.Id), nil - } resolvers["instance-pools"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { w := b.WorkspaceClient() entity, err := w.InstancePools.GetByInstancePoolName(ctx, name) @@ -104,15 +59,6 @@ func Resolvers() map[string](ResolverFunc) { return fmt.Sprint(entity.InstancePoolId), nil } - resolvers["ip-access-lists"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.IpAccessLists.GetByLabel(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.ListId), nil - } resolvers["jobs"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { w := b.WorkspaceClient() entity, err := w.Jobs.GetBySettingsName(ctx, name) @@ -149,87 +95,6 @@ func Resolvers() map[string](ResolverFunc) { return fmt.Sprint(entity.Id), nil } - resolvers["registered-models"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.RegisteredModels.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.FullName), nil - } - resolvers["repos"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Repos.GetByPath(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.Id), nil - } - resolvers["schemas"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Schemas.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.FullName), nil - } - resolvers["service-principals"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.ServicePrincipals.GetByDisplayName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.Id), nil - } - resolvers["tables"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Tables.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.TableId), nil - } - resolvers["token-management"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.TokenManagement.GetByComment(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.TokenId), nil - } - resolvers["tokens"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Tokens.GetByComment(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.TokenId), nil - } - resolvers["users"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Users.GetByUserName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.Id), nil - } - resolvers["volumes"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Volumes.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.VolumeId), nil - } resolvers["warehouses"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { w := b.WorkspaceClient() entity, err := w.Warehouses.GetByName(ctx, name) @@ -239,15 +104,6 @@ func Resolvers() map[string](ResolverFunc) { return fmt.Sprint(entity.Id), nil } - resolvers["workspace"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Workspace.GetByPath(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.ObjectId), nil - } return resolvers } From 0a9ac46f014777652dc706b387ed864f8d9dc40b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 25 Oct 2023 18:07:31 +0200 Subject: [PATCH 04/11] Update bundle/config/mutator/resolve_resource_references.go Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> --- bundle/config/mutator/resolve_resource_references.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/mutator/resolve_resource_references.go b/bundle/config/mutator/resolve_resource_references.go index f2e665a4bf..03d9c7be11 100644 --- a/bundle/config/mutator/resolve_resource_references.go +++ b/bundle/config/mutator/resolve_resource_references.go @@ -40,7 +40,7 @@ func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) lookup := v.Lookup resource, name, ok := strings.Cut(lookup, separator) if !ok { - return fmt.Errorf("incorrect lookup specified %s", lookup) + return fmt.Errorf("unexpected format for lookup: %s. Expected lookup string to be of the form :", lookup) } resolver, ok := m.resolvers[resource] From 0248e66c08b5bd5e38c6ae5f8c450a2668a68a70 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 2 Jan 2024 15:52:30 +0100 Subject: [PATCH 05/11] target override --- bundle/config/root.go | 20 +++++++++++++++---- bundle/config/target.go | 4 ++-- .../variables/env_overrides/databricks.yml | 10 ++++++++++ bundle/tests/variables_test.go | 12 +++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 32baa1a506..44344b266e 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -201,13 +201,25 @@ func (r *Root) MergeTargetOverrides(target *Target) error { if target.Variables != nil { for k, v := range target.Variables { - variable, ok := r.Variables[k] + rootVariable, ok := r.Variables[k] if !ok { return fmt.Errorf("variable %s is not defined but is assigned a value", k) } - // we only allow overrides of the default value for a variable - defaultVal := v - variable.Default = &defaultVal + + if sv, ok := v.(string); ok { + // we allow overrides of the default value for a variable + defaultVal := sv + rootVariable.Default = &defaultVal + } else if vv, ok := v.(map[string]interface{}); ok { + // we also allow overrides of the lookup value for a variable + lookup, ok := vv["lookup"] + if !ok { + return fmt.Errorf("variable %s is incorrectly defined lookup override, no 'lookup' key defined", k) + } + rootVariable.Lookup = lookup.(string) + } else { + return fmt.Errorf("variable %s is incorrectly defined in target override", k) + } } } diff --git a/bundle/config/target.go b/bundle/config/target.go index 1264430e21..c8ee358a80 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -30,10 +30,10 @@ type Target struct { Resources *Resources `json:"resources,omitempty"` - // Override default values for defined variables + // Override default values or lookup naem for defined variables // Does not permit defining new variables or redefining existing ones // in the scope of an target - Variables map[string]string `json:"variables,omitempty"` + Variables map[string]interface{} `json:"variables,omitempty"` Git Git `json:"git,omitempty"` diff --git a/bundle/tests/variables/env_overrides/databricks.yml b/bundle/tests/variables/env_overrides/databricks.yml index 2157596c38..42f78fb85e 100644 --- a/bundle/tests/variables/env_overrides/databricks.yml +++ b/bundle/tests/variables/env_overrides/databricks.yml @@ -6,6 +6,10 @@ variables: b: description: required variable + d: + description: variable with lookup + lookup: "clusters: some-cluster" + bundle: name: test bundle @@ -30,3 +34,9 @@ targets: variables: c: prod-c b: prod-b + + env-overrides-lookup: + variables: + d: + lookup: "clusters: some-named-lookup" + b: prod-b diff --git a/bundle/tests/variables_test.go b/bundle/tests/variables_test.go index 86706ebd14..f369b1ac16 100644 --- a/bundle/tests/variables_test.go +++ b/bundle/tests/variables_test.go @@ -104,3 +104,15 @@ func TestVariablesWithoutDefinition(t *testing.T) { assert.Equal(t, "foo", *b.Config.Variables["a"].Value) assert.Equal(t, "bar", *b.Config.Variables["b"].Value) } + +func TestVariablesWithTargetLookupOverrides(t *testing.T) { + b := load(t, "./variables/env_overrides") + err := bundle.Apply(context.Background(), b, bundle.Seq( + mutator.SelectTarget("env-overrides-lookup"), + mutator.SetVariables(), + interpolation.Interpolate( + interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix), + ))) + require.NoError(t, err) + assert.Equal(t, "clusters: some-test-cluster", b.Config.Variables["d"].Lookup) +} From 60953f3ef80dabd4bf1103d3309f34957514b4e8 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 2 Jan 2024 15:55:18 +0100 Subject: [PATCH 06/11] fixed formatting --- .codegen/resolvers.go.tmpl | 62 +++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.codegen/resolvers.go.tmpl b/.codegen/resolvers.go.tmpl index da8cd72ac9..a38d8ebf22 100644 --- a/.codegen/resolvers.go.tmpl +++ b/.codegen/resolvers.go.tmpl @@ -5,47 +5,47 @@ package resolvers {{ $allowlist := list "alerts" - "clusters" - "cluster-policies" - "clusters" - "dashboards" - "instance-pools" - "jobs" - "metastores" - "pipelines" - "queries" - "warehouses" + "clusters" + "cluster-policies" + "clusters" + "dashboards" + "instance-pools" + "jobs" + "metastores" + "pipelines" + "queries" + "warehouses" }} import ( "context" - "fmt" + "fmt" "github.com/databricks/cli/bundle" ) type ResolverFunc func(ctx context.Context, b *bundle.Bundle, name string) (string, error) func Resolvers() map[string](ResolverFunc) { - resolvers := make(map[string](ResolverFunc), 0) - {{range .Services -}} - {{- if in $allowlist .KebabName -}} - {{- if not .IsAccounts -}} - {{- if and .List .List.GetByName }} - resolvers["{{.KebabName}}"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.{{.PascalName}}.GetBy{{range .List.NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity{{ template "field-path" .List.NamedIdMap.IdPath }}), nil - } - {{- end -}} - {{- end -}} - {{- end -}} - {{- end}} - - return resolvers + resolvers := make(map[string](ResolverFunc), 0) + {{range .Services -}} + {{- if in $allowlist .KebabName -}} + {{- if not .IsAccounts -}} + {{- if and .List .List.GetByName }} + resolvers["{{.KebabName}}"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { + w := b.WorkspaceClient() + entity, err := w.{{.PascalName}}.GetBy{{range .List.NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity{{ template "field-path" .List.NamedIdMap.IdPath }}), nil + } + {{- end -}} + {{- end -}} + {{- end -}} + {{- end}} + + return resolvers } From 766ba5bb252e78ae865438b1c87d1e5f15979e08 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 2 Jan 2024 16:00:35 +0100 Subject: [PATCH 07/11] fixed typo --- bundle/tests/variables/env_overrides/databricks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/tests/variables/env_overrides/databricks.yml b/bundle/tests/variables/env_overrides/databricks.yml index 42f78fb85e..c74504a63d 100644 --- a/bundle/tests/variables/env_overrides/databricks.yml +++ b/bundle/tests/variables/env_overrides/databricks.yml @@ -38,5 +38,5 @@ targets: env-overrides-lookup: variables: d: - lookup: "clusters: some-named-lookup" + lookup: "clusters: some-test-cluster" b: prod-b From 485bd05693e84bf50a748c74bd52eb1ce80ca24a Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 4 Jan 2024 14:44:32 +0100 Subject: [PATCH 08/11] structured lookup field --- .codegen.json | 2 +- .codegen/lookup.go.tmpl | 122 ++++++++ .codegen/resolvers.go.tmpl | 54 ---- .gitattributes | 2 +- .../mutator/resolve_resource_references.go | 29 +- .../resolve_resource_references_test.go | 44 ++- bundle/config/mutator/set_variables.go | 2 +- bundle/config/root.go | 2 +- bundle/config/target.go | 4 +- bundle/config/variable/lookup.go | 277 ++++++++++++++++++ bundle/config/variable/variable.go | 2 +- bundle/resolvers/resolvers.go | 109 ------- .../variables/env_overrides/databricks.yml | 6 +- bundle/tests/variables_test.go | 2 +- 14 files changed, 433 insertions(+), 224 deletions(-) create mode 100644 .codegen/lookup.go.tmpl delete mode 100644 .codegen/resolvers.go.tmpl create mode 100755 bundle/config/variable/lookup.go delete mode 100755 bundle/resolvers/resolvers.go diff --git a/.codegen.json b/.codegen.json index ee8495faf4..8cb42b415c 100644 --- a/.codegen.json +++ b/.codegen.json @@ -6,7 +6,7 @@ "batch": { ".codegen/cmds-workspace.go.tmpl": "cmd/workspace/cmd.go", ".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go", - ".codegen/resolvers.go.tmpl": "bundle/resolvers/resolvers.go" + ".codegen/lookup.go.tmpl": "bundle/config/variable/lookup.go" }, "toolchain": { "required": ["go"], diff --git a/.codegen/lookup.go.tmpl b/.codegen/lookup.go.tmpl new file mode 100644 index 0000000000..a19e936791 --- /dev/null +++ b/.codegen/lookup.go.tmpl @@ -0,0 +1,122 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package variable + +{{ $allowlist := + list + "alerts" + "clusters" + "cluster-policies" + "clusters" + "dashboards" + "instance-pools" + "jobs" + "metastores" + "pipelines" + "queries" + "warehouses" +}} + +import ( + "context" + "fmt" + + "github.com/databricks/databricks-sdk-go" +) + +type Lookup struct { + {{range .Services -}} + {{- if in $allowlist .KebabName -}} + {{.Singular.PascalName}} string `json:"{{.Singular.KebabName}},omitempty"` + + {{end}} + {{- end}} +} + +func LookupFromMap(m map[string]interface{}) *Lookup { + l := &Lookup{} + {{range .Services -}} + {{- if in $allowlist .KebabName -}} + if v, ok := m["{{.Singular.KebabName}}"]; ok { + l.{{.Singular.PascalName}} = v.(string) + } + {{end -}} + {{- end}} + return l +} + +func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) { + if err := l.validate(); err != nil { + return "", err + } + + resolvers := resolvers() + {{range .Services -}} + {{- if in $allowlist .KebabName -}} + if l.{{.Singular.PascalName}} != "" { + return resolvers["{{.Singular.KebabName}}"](ctx, w, l.{{.Singular.PascalName}}) + } + {{end -}} + {{- end}} + + return "", fmt.Errorf("no valid lookup fields provided") +} + +func (l *Lookup) String() string { + {{range .Services -}} + {{- if in $allowlist .KebabName -}} + if l.{{.Singular.PascalName}} != "" { + return fmt.Sprintf("{{.Singular.KebabName}}: %s", l.{{.Singular.PascalName}}) + } + {{end -}} + {{- end}} + return "" +} + +func (l *Lookup) validate() error { + // Validate that only one field is set + count := 0 + {{range .Services -}} + {{- if in $allowlist .KebabName -}} + if l.{{.Singular.PascalName}} != "" { + count++ + } + {{end -}} + {{- end}} + + if count != 1 { + return fmt.Errorf("exactly one lookup field must be provided") + } + + if strings.Contains(l.String(), "${var") { + return fmt.Errorf("lookup fields cannot contain variable references") + } + + return nil +} + + +type resolverFunc func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) + +func resolvers() map[string](resolverFunc) { + resolvers := make(map[string](resolverFunc), 0) + {{range .Services -}} + {{- if in $allowlist .KebabName -}} + resolvers["{{.Singular.KebabName}}"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.{{.PascalName}}.GetBy{{range .List.NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity{{ template "field-path" .List.NamedIdMap.IdPath }}), nil + } + {{end -}} + {{- end}} + + return resolvers +} + + +{{- define "field-path" -}} + {{- range .}}.{{.PascalName}}{{end}} +{{- end -}} diff --git a/.codegen/resolvers.go.tmpl b/.codegen/resolvers.go.tmpl deleted file mode 100644 index a38d8ebf22..0000000000 --- a/.codegen/resolvers.go.tmpl +++ /dev/null @@ -1,54 +0,0 @@ -// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. - -package resolvers - -{{ $allowlist := - list - "alerts" - "clusters" - "cluster-policies" - "clusters" - "dashboards" - "instance-pools" - "jobs" - "metastores" - "pipelines" - "queries" - "warehouses" -}} - -import ( - "context" - "fmt" - "github.com/databricks/cli/bundle" -) - -type ResolverFunc func(ctx context.Context, b *bundle.Bundle, name string) (string, error) - -func Resolvers() map[string](ResolverFunc) { - resolvers := make(map[string](ResolverFunc), 0) - {{range .Services -}} - {{- if in $allowlist .KebabName -}} - {{- if not .IsAccounts -}} - {{- if and .List .List.GetByName }} - resolvers["{{.KebabName}}"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.{{.PascalName}}.GetBy{{range .List.NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity{{ template "field-path" .List.NamedIdMap.IdPath }}), nil - } - {{- end -}} - {{- end -}} - {{- end -}} - {{- end}} - - return resolvers -} - - -{{- define "field-path" -}} - {{- range .}}.{{.PascalName}}{{end}} -{{- end -}} diff --git a/.gitattributes b/.gitattributes index 572b91cfc0..7826790509 100755 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,4 @@ -bundle/resolvers/resolvers.go linguist-generated=true +bundle/config/variable/lookup.go linguist-generated=true cmd/account/access-control/access-control.go linguist-generated=true cmd/account/billable-usage/billable-usage.go linguist-generated=true cmd/account/budgets/budgets.go linguist-generated=true diff --git a/bundle/config/mutator/resolve_resource_references.go b/bundle/config/mutator/resolve_resource_references.go index 03d9c7be11..6fad5aae09 100644 --- a/bundle/config/mutator/resolve_resource_references.go +++ b/bundle/config/mutator/resolve_resource_references.go @@ -3,24 +3,16 @@ package mutator import ( "context" "fmt" - "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/resolvers" "github.com/databricks/cli/libs/log" "golang.org/x/sync/errgroup" ) -const separator string = ":" - -type resolveResourceReferences struct { - resolvers map[string]resolvers.ResolverFunc -} +type resolveResourceReferences struct{} func ResolveResourceReferences() bundle.Mutator { - return &resolveResourceReferences{ - resolvers: resolvers.Resolvers(), - } + return &resolveResourceReferences{} } func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) error { @@ -28,7 +20,7 @@ func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) for k := range b.Config.Variables { v := b.Config.Variables[k] - if v.Lookup == "" { + if v.Lookup == nil { continue } @@ -37,21 +29,10 @@ func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) continue } - lookup := v.Lookup - resource, name, ok := strings.Cut(lookup, separator) - if !ok { - return fmt.Errorf("unexpected format for lookup: %s. Expected lookup string to be of the form :", lookup) - } - - resolver, ok := m.resolvers[resource] - if !ok { - return fmt.Errorf("unable to resolve resource reference %s, no resolvers for %s", lookup, resource) - } - errs.Go(func() error { - id, err := resolver(errCtx, b, name) + id, err := v.Lookup.Resolve(errCtx, b.WorkspaceClient()) if err != nil { - return fmt.Errorf("failed to resolve %s reference %s, err: %w", resource, lookup, err) + return fmt.Errorf("failed to resolve %s, err: %w", v.Lookup, err) } v.Set(id) diff --git a/bundle/config/mutator/resolve_resource_references_test.go b/bundle/config/mutator/resolve_resource_references_test.go index 2b6793a0c0..f3e37afc49 100644 --- a/bundle/config/mutator/resolve_resource_references_test.go +++ b/bundle/config/mutator/resolve_resource_references_test.go @@ -119,17 +119,21 @@ func (MockClusterService) UpdatePermissions(ctx context.Context, request compute } func TestResolveClusterReference(t *testing.T) { - clusterRef1 := "clusters:Some Custom Cluster" - clusterRef2 := "clusters:Some Other Name" + clusterRef1 := "Some Custom Cluster" + clusterRef2 := "Some Other Name" justString := "random string" b := &bundle.Bundle{ Config: config.Root{ Variables: map[string]*variable.Variable{ "my-cluster-id-1": { - Lookup: clusterRef1, + Lookup: &variable.Lookup{ + Cluster: clusterRef1, + }, }, "my-cluster-id-2": { - Lookup: clusterRef2, + Lookup: &variable.Lookup{ + Cluster: clusterRef2, + }, }, "some-variable": { Value: &justString, @@ -147,13 +151,15 @@ func TestResolveClusterReference(t *testing.T) { } func TestResolveNonExistentClusterReference(t *testing.T) { - clusterRef := "clusters:Random" + clusterRef := "Random" justString := "random string" b := &bundle.Bundle{ Config: config.Root{ Variables: map[string]*variable.Variable{ "my-cluster-id": { - Lookup: clusterRef, + Lookup: &variable.Lookup{ + Cluster: clusterRef, + }, }, "some-variable": { Value: &justString, @@ -165,34 +171,18 @@ func TestResolveNonExistentClusterReference(t *testing.T) { b.WorkspaceClient().Clusters.WithImpl(MockClusterService{}) err := bundle.Apply(context.Background(), b, ResolveResourceReferences()) - require.ErrorContains(t, err, "failed to resolve clusters reference clusters:Random, err: ClusterDetails named 'Random' does not exist") -} - -func TestResolveNonExistentResourceType(t *testing.T) { - clusterRef := "donotexist:Random" - b := &bundle.Bundle{ - Config: config.Root{ - Variables: map[string]*variable.Variable{ - "my-cluster-id": { - Lookup: clusterRef, - }, - }, - }, - } - - b.WorkspaceClient().Clusters.WithImpl(MockClusterService{}) - - err := bundle.Apply(context.Background(), b, ResolveResourceReferences()) - require.ErrorContains(t, err, "unable to resolve resource reference donotexist:Random, no resolvers for donotexist") + require.ErrorContains(t, err, "failed to resolve cluster: Random, err: ClusterDetails named 'Random' does not exist") } func TestNoLookupIfVariableIsSet(t *testing.T) { - clusterRef := "donotexist:Random" + clusterRef := "donotexist" b := &bundle.Bundle{ Config: config.Root{ Variables: map[string]*variable.Variable{ "my-cluster-id": { - Lookup: clusterRef, + Lookup: &variable.Lookup{ + Cluster: clusterRef, + }, }, }, }, diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index c59101479c..3b9ac8ae76 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -48,7 +48,7 @@ func setVariable(ctx context.Context, v *variable.Variable, name string) error { // case: Defined a variable for named lookup for a resource // It will be resolved later in ResolveResourceReferences mutator - if v.Lookup != "" { + if v.Lookup != nil { return nil } diff --git a/bundle/config/root.go b/bundle/config/root.go index 44344b266e..99168d9636 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -216,7 +216,7 @@ func (r *Root) MergeTargetOverrides(target *Target) error { if !ok { return fmt.Errorf("variable %s is incorrectly defined lookup override, no 'lookup' key defined", k) } - rootVariable.Lookup = lookup.(string) + rootVariable.Lookup = variable.LookupFromMap(lookup.(map[string]interface{})) } else { return fmt.Errorf("variable %s is incorrectly defined in target override", k) } diff --git a/bundle/config/target.go b/bundle/config/target.go index c8ee358a80..158f256060 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -30,10 +30,10 @@ type Target struct { Resources *Resources `json:"resources,omitempty"` - // Override default values or lookup naem for defined variables + // Override default values or lookup name for defined variables // Does not permit defining new variables or redefining existing ones // in the scope of an target - Variables map[string]interface{} `json:"variables,omitempty"` + Variables map[string]any `json:"variables,omitempty"` Git Git `json:"git,omitempty"` diff --git a/bundle/config/variable/lookup.go b/bundle/config/variable/lookup.go new file mode 100755 index 0000000000..582f732afc --- /dev/null +++ b/bundle/config/variable/lookup.go @@ -0,0 +1,277 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package variable + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/databricks-sdk-go" +) + +type Lookup struct { + Alert string `json:"alert,omitempty"` + + ClusterPolicy string `json:"cluster-policy,omitempty"` + + Cluster string `json:"cluster,omitempty"` + + Dashboard string `json:"dashboard,omitempty"` + + InstancePool string `json:"instance-pool,omitempty"` + + Job string `json:"job,omitempty"` + + Metastore string `json:"metastore,omitempty"` + + Pipeline string `json:"pipeline,omitempty"` + + Query string `json:"query,omitempty"` + + Warehouse string `json:"warehouse,omitempty"` +} + +func LookupFromMap(m map[string]interface{}) *Lookup { + l := &Lookup{} + if v, ok := m["alert"]; ok { + l.Alert = v.(string) + } + if v, ok := m["cluster-policy"]; ok { + l.ClusterPolicy = v.(string) + } + if v, ok := m["cluster"]; ok { + l.Cluster = v.(string) + } + if v, ok := m["dashboard"]; ok { + l.Dashboard = v.(string) + } + if v, ok := m["instance-pool"]; ok { + l.InstancePool = v.(string) + } + if v, ok := m["job"]; ok { + l.Job = v.(string) + } + if v, ok := m["metastore"]; ok { + l.Metastore = v.(string) + } + if v, ok := m["pipeline"]; ok { + l.Pipeline = v.(string) + } + if v, ok := m["query"]; ok { + l.Query = v.(string) + } + if v, ok := m["warehouse"]; ok { + l.Warehouse = v.(string) + } + + return l +} + +func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) { + if err := l.validate(); err != nil { + return "", err + } + + resolvers := resolvers() + if l.Alert != "" { + return resolvers["alert"](ctx, w, l.Alert) + } + if l.ClusterPolicy != "" { + return resolvers["cluster-policy"](ctx, w, l.ClusterPolicy) + } + if l.Cluster != "" { + return resolvers["cluster"](ctx, w, l.Cluster) + } + if l.Dashboard != "" { + return resolvers["dashboard"](ctx, w, l.Dashboard) + } + if l.InstancePool != "" { + return resolvers["instance-pool"](ctx, w, l.InstancePool) + } + if l.Job != "" { + return resolvers["job"](ctx, w, l.Job) + } + if l.Metastore != "" { + return resolvers["metastore"](ctx, w, l.Metastore) + } + if l.Pipeline != "" { + return resolvers["pipeline"](ctx, w, l.Pipeline) + } + if l.Query != "" { + return resolvers["query"](ctx, w, l.Query) + } + if l.Warehouse != "" { + return resolvers["warehouse"](ctx, w, l.Warehouse) + } + + return "", fmt.Errorf("no valid lookup fields provided") +} + +func (l *Lookup) String() string { + if l.Alert != "" { + return fmt.Sprintf("alert: %s", l.Alert) + } + if l.ClusterPolicy != "" { + return fmt.Sprintf("cluster-policy: %s", l.ClusterPolicy) + } + if l.Cluster != "" { + return fmt.Sprintf("cluster: %s", l.Cluster) + } + if l.Dashboard != "" { + return fmt.Sprintf("dashboard: %s", l.Dashboard) + } + if l.InstancePool != "" { + return fmt.Sprintf("instance-pool: %s", l.InstancePool) + } + if l.Job != "" { + return fmt.Sprintf("job: %s", l.Job) + } + if l.Metastore != "" { + return fmt.Sprintf("metastore: %s", l.Metastore) + } + if l.Pipeline != "" { + return fmt.Sprintf("pipeline: %s", l.Pipeline) + } + if l.Query != "" { + return fmt.Sprintf("query: %s", l.Query) + } + if l.Warehouse != "" { + return fmt.Sprintf("warehouse: %s", l.Warehouse) + } + + return "" +} + +func (l *Lookup) validate() error { + // Validate that only one field is set + count := 0 + if l.Alert != "" { + count++ + } + if l.ClusterPolicy != "" { + count++ + } + if l.Cluster != "" { + count++ + } + if l.Dashboard != "" { + count++ + } + if l.InstancePool != "" { + count++ + } + if l.Job != "" { + count++ + } + if l.Metastore != "" { + count++ + } + if l.Pipeline != "" { + count++ + } + if l.Query != "" { + count++ + } + if l.Warehouse != "" { + count++ + } + + if count != 1 { + return fmt.Errorf("exactly one lookup field must be provided") + } + + if strings.Contains(l.String(), "${var") { + return fmt.Errorf("lookup fields cannot contain variable references") + } + + return nil +} + +type resolverFunc func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) + +func resolvers() map[string](resolverFunc) { + resolvers := make(map[string](resolverFunc), 0) + resolvers["alert"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.Alerts.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["cluster-policy"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.ClusterPolicies.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.PolicyId), nil + } + resolvers["cluster"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.Clusters.GetByClusterName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.ClusterId), nil + } + resolvers["dashboard"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.Dashboards.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["instance-pool"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.InstancePools.GetByInstancePoolName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.InstancePoolId), nil + } + resolvers["job"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.Jobs.GetBySettingsName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.JobId), nil + } + resolvers["metastore"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.Metastores.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.MetastoreId), nil + } + resolvers["pipeline"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.Pipelines.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.PipelineId), nil + } + resolvers["query"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.Queries.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + resolvers["warehouse"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.Warehouses.GetByName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } + + return resolvers +} diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index f2a2dd8e8b..964ea0dea8 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -29,7 +29,7 @@ type Variable struct { // Format: ":" // The value of this field will be used to lookup the resource by name // And assign the value of the variable to ID of the resource found. - Lookup string `json:"lookup,omitempty"` + Lookup *Lookup `json:"lookup,omitempty"` } // True if the variable has been assigned a default value. Variables without a diff --git a/bundle/resolvers/resolvers.go b/bundle/resolvers/resolvers.go deleted file mode 100755 index 14a5a69966..0000000000 --- a/bundle/resolvers/resolvers.go +++ /dev/null @@ -1,109 +0,0 @@ -// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. - -package resolvers - -import ( - "context" - "fmt" - - "github.com/databricks/cli/bundle" -) - -type ResolverFunc func(ctx context.Context, b *bundle.Bundle, name string) (string, error) - -func Resolvers() map[string](ResolverFunc) { - resolvers := make(map[string](ResolverFunc), 0) - - resolvers["alerts"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Alerts.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.Id), nil - } - resolvers["cluster-policies"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.ClusterPolicies.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.PolicyId), nil - } - resolvers["clusters"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Clusters.GetByClusterName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.ClusterId), nil - } - resolvers["dashboards"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Dashboards.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.Id), nil - } - resolvers["instance-pools"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.InstancePools.GetByInstancePoolName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.InstancePoolId), nil - } - resolvers["jobs"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Jobs.GetBySettingsName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.JobId), nil - } - resolvers["metastores"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Metastores.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.MetastoreId), nil - } - resolvers["pipelines"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Pipelines.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.PipelineId), nil - } - resolvers["queries"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Queries.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.Id), nil - } - resolvers["warehouses"] = func(ctx context.Context, b *bundle.Bundle, name string) (string, error) { - w := b.WorkspaceClient() - entity, err := w.Warehouses.GetByName(ctx, name) - if err != nil { - return "", err - } - - return fmt.Sprint(entity.Id), nil - } - - return resolvers -} diff --git a/bundle/tests/variables/env_overrides/databricks.yml b/bundle/tests/variables/env_overrides/databricks.yml index c74504a63d..f7e4c3ceae 100644 --- a/bundle/tests/variables/env_overrides/databricks.yml +++ b/bundle/tests/variables/env_overrides/databricks.yml @@ -8,7 +8,8 @@ variables: d: description: variable with lookup - lookup: "clusters: some-cluster" + lookup: + cluster: some-cluster bundle: name: test bundle @@ -38,5 +39,6 @@ targets: env-overrides-lookup: variables: d: - lookup: "clusters: some-test-cluster" + lookup: + cluster: some-test-cluster b: prod-b diff --git a/bundle/tests/variables_test.go b/bundle/tests/variables_test.go index f369b1ac16..47a1861250 100644 --- a/bundle/tests/variables_test.go +++ b/bundle/tests/variables_test.go @@ -114,5 +114,5 @@ func TestVariablesWithTargetLookupOverrides(t *testing.T) { interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix), ))) require.NoError(t, err) - assert.Equal(t, "clusters: some-test-cluster", b.Config.Variables["d"].Lookup) + assert.Equal(t, "some-test-cluster", b.Config.Variables["d"].Lookup.Cluster) } From af687ad5a61a92c06b92ede064b0d165caf09c61 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 4 Jan 2024 14:49:12 +0100 Subject: [PATCH 09/11] updated comment --- bundle/config/variable/variable.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index 964ea0dea8..9057f1cb95 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -25,8 +25,6 @@ type Variable struct { // is required Value *string `json:"value,omitempty" bundle:"readonly"` - // A string value that represents a reference to the remote resource by name - // Format: ":" // The value of this field will be used to lookup the resource by name // And assign the value of the variable to ID of the resource found. Lookup *Lookup `json:"lookup,omitempty"` From 8288b7a2a5aea2e56f78d0aca37890f486810c14 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 4 Jan 2024 16:05:51 +0100 Subject: [PATCH 10/11] fixes --- .codegen/lookup.go.tmpl | 16 +- .../mutator/resolve_resource_references.go | 2 +- bundle/config/root.go | 4 +- bundle/config/variable/lookup.go | 142 +++++++++--------- bundle/tests/variables_test.go | 2 +- 5 files changed, 83 insertions(+), 83 deletions(-) diff --git a/.codegen/lookup.go.tmpl b/.codegen/lookup.go.tmpl index a19e936791..3e609978c3 100644 --- a/.codegen/lookup.go.tmpl +++ b/.codegen/lookup.go.tmpl @@ -27,18 +27,18 @@ import ( type Lookup struct { {{range .Services -}} {{- if in $allowlist .KebabName -}} - {{.Singular.PascalName}} string `json:"{{.Singular.KebabName}},omitempty"` + {{.Singular.CamelName}} string {{end}} {{- end}} } -func LookupFromMap(m map[string]interface{}) *Lookup { +func LookupFromMap(m map[string]any) *Lookup { l := &Lookup{} {{range .Services -}} {{- if in $allowlist .KebabName -}} if v, ok := m["{{.Singular.KebabName}}"]; ok { - l.{{.Singular.PascalName}} = v.(string) + l.{{.Singular.CamelName}} = v.(string) } {{end -}} {{- end}} @@ -53,8 +53,8 @@ func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (st resolvers := resolvers() {{range .Services -}} {{- if in $allowlist .KebabName -}} - if l.{{.Singular.PascalName}} != "" { - return resolvers["{{.Singular.KebabName}}"](ctx, w, l.{{.Singular.PascalName}}) + if l.{{.Singular.CamelName}} != "" { + return resolvers["{{.Singular.KebabName}}"](ctx, w, l.{{.Singular.CamelName}}) } {{end -}} {{- end}} @@ -65,8 +65,8 @@ func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (st func (l *Lookup) String() string { {{range .Services -}} {{- if in $allowlist .KebabName -}} - if l.{{.Singular.PascalName}} != "" { - return fmt.Sprintf("{{.Singular.KebabName}}: %s", l.{{.Singular.PascalName}}) + if l.{{.Singular.CamelName}} != "" { + return fmt.Sprintf("{{.Singular.KebabName}}: %s", l.{{.Singular.CamelName}}) } {{end -}} {{- end}} @@ -78,7 +78,7 @@ func (l *Lookup) validate() error { count := 0 {{range .Services -}} {{- if in $allowlist .KebabName -}} - if l.{{.Singular.PascalName}} != "" { + if l.{{.Singular.CamelName}} != "" { count++ } {{end -}} diff --git a/bundle/config/mutator/resolve_resource_references.go b/bundle/config/mutator/resolve_resource_references.go index 6fad5aae09..7a7462ab9d 100644 --- a/bundle/config/mutator/resolve_resource_references.go +++ b/bundle/config/mutator/resolve_resource_references.go @@ -20,7 +20,7 @@ func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) for k := range b.Config.Variables { v := b.Config.Variables[k] - if v.Lookup == nil { + if v == nil || v.Lookup == nil { continue } diff --git a/bundle/config/root.go b/bundle/config/root.go index 99168d9636..94cc0b177b 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -210,13 +210,13 @@ func (r *Root) MergeTargetOverrides(target *Target) error { // we allow overrides of the default value for a variable defaultVal := sv rootVariable.Default = &defaultVal - } else if vv, ok := v.(map[string]interface{}); ok { + } else if vv, ok := v.(map[string]any); ok { // we also allow overrides of the lookup value for a variable lookup, ok := vv["lookup"] if !ok { return fmt.Errorf("variable %s is incorrectly defined lookup override, no 'lookup' key defined", k) } - rootVariable.Lookup = variable.LookupFromMap(lookup.(map[string]interface{})) + rootVariable.Lookup = variable.LookupFromMap(lookup.(map[string]any)) } else { return fmt.Errorf("variable %s is incorrectly defined in target override", k) } diff --git a/bundle/config/variable/lookup.go b/bundle/config/variable/lookup.go index 582f732afc..00c61376ee 100755 --- a/bundle/config/variable/lookup.go +++ b/bundle/config/variable/lookup.go @@ -11,58 +11,58 @@ import ( ) type Lookup struct { - Alert string `json:"alert,omitempty"` + alert string - ClusterPolicy string `json:"cluster-policy,omitempty"` + clusterPolicy string - Cluster string `json:"cluster,omitempty"` + cluster string - Dashboard string `json:"dashboard,omitempty"` + dashboard string - InstancePool string `json:"instance-pool,omitempty"` + instancePool string - Job string `json:"job,omitempty"` + job string - Metastore string `json:"metastore,omitempty"` + metastore string - Pipeline string `json:"pipeline,omitempty"` + pipeline string - Query string `json:"query,omitempty"` + query string - Warehouse string `json:"warehouse,omitempty"` + warehouse string } -func LookupFromMap(m map[string]interface{}) *Lookup { +func LookupFromMap(m map[string]any) *Lookup { l := &Lookup{} if v, ok := m["alert"]; ok { - l.Alert = v.(string) + l.alert = v.(string) } if v, ok := m["cluster-policy"]; ok { - l.ClusterPolicy = v.(string) + l.clusterPolicy = v.(string) } if v, ok := m["cluster"]; ok { - l.Cluster = v.(string) + l.cluster = v.(string) } if v, ok := m["dashboard"]; ok { - l.Dashboard = v.(string) + l.dashboard = v.(string) } if v, ok := m["instance-pool"]; ok { - l.InstancePool = v.(string) + l.instancePool = v.(string) } if v, ok := m["job"]; ok { - l.Job = v.(string) + l.job = v.(string) } if v, ok := m["metastore"]; ok { - l.Metastore = v.(string) + l.metastore = v.(string) } if v, ok := m["pipeline"]; ok { - l.Pipeline = v.(string) + l.pipeline = v.(string) } if v, ok := m["query"]; ok { - l.Query = v.(string) + l.query = v.(string) } if v, ok := m["warehouse"]; ok { - l.Warehouse = v.(string) + l.warehouse = v.(string) } return l @@ -74,70 +74,70 @@ func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (st } resolvers := resolvers() - if l.Alert != "" { - return resolvers["alert"](ctx, w, l.Alert) + if l.alert != "" { + return resolvers["alert"](ctx, w, l.alert) } - if l.ClusterPolicy != "" { - return resolvers["cluster-policy"](ctx, w, l.ClusterPolicy) + if l.clusterPolicy != "" { + return resolvers["cluster-policy"](ctx, w, l.clusterPolicy) } - if l.Cluster != "" { - return resolvers["cluster"](ctx, w, l.Cluster) + if l.cluster != "" { + return resolvers["cluster"](ctx, w, l.cluster) } - if l.Dashboard != "" { - return resolvers["dashboard"](ctx, w, l.Dashboard) + if l.dashboard != "" { + return resolvers["dashboard"](ctx, w, l.dashboard) } - if l.InstancePool != "" { - return resolvers["instance-pool"](ctx, w, l.InstancePool) + if l.instancePool != "" { + return resolvers["instance-pool"](ctx, w, l.instancePool) } - if l.Job != "" { - return resolvers["job"](ctx, w, l.Job) + if l.job != "" { + return resolvers["job"](ctx, w, l.job) } - if l.Metastore != "" { - return resolvers["metastore"](ctx, w, l.Metastore) + if l.metastore != "" { + return resolvers["metastore"](ctx, w, l.metastore) } - if l.Pipeline != "" { - return resolvers["pipeline"](ctx, w, l.Pipeline) + if l.pipeline != "" { + return resolvers["pipeline"](ctx, w, l.pipeline) } - if l.Query != "" { - return resolvers["query"](ctx, w, l.Query) + if l.query != "" { + return resolvers["query"](ctx, w, l.query) } - if l.Warehouse != "" { - return resolvers["warehouse"](ctx, w, l.Warehouse) + if l.warehouse != "" { + return resolvers["warehouse"](ctx, w, l.warehouse) } return "", fmt.Errorf("no valid lookup fields provided") } func (l *Lookup) String() string { - if l.Alert != "" { - return fmt.Sprintf("alert: %s", l.Alert) + if l.alert != "" { + return fmt.Sprintf("alert: %s", l.alert) } - if l.ClusterPolicy != "" { - return fmt.Sprintf("cluster-policy: %s", l.ClusterPolicy) + if l.clusterPolicy != "" { + return fmt.Sprintf("cluster-policy: %s", l.clusterPolicy) } - if l.Cluster != "" { - return fmt.Sprintf("cluster: %s", l.Cluster) + if l.cluster != "" { + return fmt.Sprintf("cluster: %s", l.cluster) } - if l.Dashboard != "" { - return fmt.Sprintf("dashboard: %s", l.Dashboard) + if l.dashboard != "" { + return fmt.Sprintf("dashboard: %s", l.dashboard) } - if l.InstancePool != "" { - return fmt.Sprintf("instance-pool: %s", l.InstancePool) + if l.instancePool != "" { + return fmt.Sprintf("instance-pool: %s", l.instancePool) } - if l.Job != "" { - return fmt.Sprintf("job: %s", l.Job) + if l.job != "" { + return fmt.Sprintf("job: %s", l.job) } - if l.Metastore != "" { - return fmt.Sprintf("metastore: %s", l.Metastore) + if l.metastore != "" { + return fmt.Sprintf("metastore: %s", l.metastore) } - if l.Pipeline != "" { - return fmt.Sprintf("pipeline: %s", l.Pipeline) + if l.pipeline != "" { + return fmt.Sprintf("pipeline: %s", l.pipeline) } - if l.Query != "" { - return fmt.Sprintf("query: %s", l.Query) + if l.query != "" { + return fmt.Sprintf("query: %s", l.query) } - if l.Warehouse != "" { - return fmt.Sprintf("warehouse: %s", l.Warehouse) + if l.warehouse != "" { + return fmt.Sprintf("warehouse: %s", l.warehouse) } return "" @@ -146,34 +146,34 @@ func (l *Lookup) String() string { func (l *Lookup) validate() error { // Validate that only one field is set count := 0 - if l.Alert != "" { + if l.alert != "" { count++ } - if l.ClusterPolicy != "" { + if l.clusterPolicy != "" { count++ } - if l.Cluster != "" { + if l.cluster != "" { count++ } - if l.Dashboard != "" { + if l.dashboard != "" { count++ } - if l.InstancePool != "" { + if l.instancePool != "" { count++ } - if l.Job != "" { + if l.job != "" { count++ } - if l.Metastore != "" { + if l.metastore != "" { count++ } - if l.Pipeline != "" { + if l.pipeline != "" { count++ } - if l.Query != "" { + if l.query != "" { count++ } - if l.Warehouse != "" { + if l.warehouse != "" { count++ } diff --git a/bundle/tests/variables_test.go b/bundle/tests/variables_test.go index 47a1861250..6579dcccec 100644 --- a/bundle/tests/variables_test.go +++ b/bundle/tests/variables_test.go @@ -114,5 +114,5 @@ func TestVariablesWithTargetLookupOverrides(t *testing.T) { interpolation.IncludeLookupsInPath(variable.VariableReferencePrefix), ))) require.NoError(t, err) - assert.Equal(t, "some-test-cluster", b.Config.Variables["d"].Lookup.Cluster) + assert.Equal(t, "cluster: some-test-cluster", b.Config.Variables["d"].Lookup.String()) } From d655b67c044571f06bbae831eedf64ccea451a7f Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 4 Jan 2024 21:59:12 +0100 Subject: [PATCH 11/11] added service-principals --- .codegen/lookup.go.tmpl | 15 +-- bundle/config/variable/lookup.go | 162 ++++++++++++++++++------------- 2 files changed, 100 insertions(+), 77 deletions(-) diff --git a/.codegen/lookup.go.tmpl b/.codegen/lookup.go.tmpl index 3e609978c3..4200edc6dc 100644 --- a/.codegen/lookup.go.tmpl +++ b/.codegen/lookup.go.tmpl @@ -13,6 +13,7 @@ package variable "jobs" "metastores" "pipelines" + "service-principals" "queries" "warehouses" }} @@ -27,7 +28,7 @@ import ( type Lookup struct { {{range .Services -}} {{- if in $allowlist .KebabName -}} - {{.Singular.CamelName}} string + {{.Singular.PascalName}} string `json:"{{.Singular.SnakeName}},omitempty"` {{end}} {{- end}} @@ -38,7 +39,7 @@ func LookupFromMap(m map[string]any) *Lookup { {{range .Services -}} {{- if in $allowlist .KebabName -}} if v, ok := m["{{.Singular.KebabName}}"]; ok { - l.{{.Singular.CamelName}} = v.(string) + l.{{.Singular.PascalName}} = v.(string) } {{end -}} {{- end}} @@ -53,8 +54,8 @@ func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (st resolvers := resolvers() {{range .Services -}} {{- if in $allowlist .KebabName -}} - if l.{{.Singular.CamelName}} != "" { - return resolvers["{{.Singular.KebabName}}"](ctx, w, l.{{.Singular.CamelName}}) + if l.{{.Singular.PascalName}} != "" { + return resolvers["{{.Singular.KebabName}}"](ctx, w, l.{{.Singular.PascalName}}) } {{end -}} {{- end}} @@ -65,8 +66,8 @@ func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (st func (l *Lookup) String() string { {{range .Services -}} {{- if in $allowlist .KebabName -}} - if l.{{.Singular.CamelName}} != "" { - return fmt.Sprintf("{{.Singular.KebabName}}: %s", l.{{.Singular.CamelName}}) + if l.{{.Singular.PascalName}} != "" { + return fmt.Sprintf("{{.Singular.KebabName}}: %s", l.{{.Singular.PascalName}}) } {{end -}} {{- end}} @@ -78,7 +79,7 @@ func (l *Lookup) validate() error { count := 0 {{range .Services -}} {{- if in $allowlist .KebabName -}} - if l.{{.Singular.CamelName}} != "" { + if l.{{.Singular.PascalName}} != "" { count++ } {{end -}} diff --git a/bundle/config/variable/lookup.go b/bundle/config/variable/lookup.go index 00c61376ee..1e029dabc0 100755 --- a/bundle/config/variable/lookup.go +++ b/bundle/config/variable/lookup.go @@ -11,58 +11,63 @@ import ( ) type Lookup struct { - alert string + Alert string `json:"alert,omitempty"` - clusterPolicy string + ClusterPolicy string `json:"cluster_policy,omitempty"` - cluster string + Cluster string `json:"cluster,omitempty"` - dashboard string + Dashboard string `json:"dashboard,omitempty"` - instancePool string + InstancePool string `json:"instance_pool,omitempty"` - job string + Job string `json:"job,omitempty"` - metastore string + Metastore string `json:"metastore,omitempty"` - pipeline string + Pipeline string `json:"pipeline,omitempty"` - query string + Query string `json:"query,omitempty"` - warehouse string + ServicePrincipal string `json:"service_principal,omitempty"` + + Warehouse string `json:"warehouse,omitempty"` } func LookupFromMap(m map[string]any) *Lookup { l := &Lookup{} if v, ok := m["alert"]; ok { - l.alert = v.(string) + l.Alert = v.(string) } if v, ok := m["cluster-policy"]; ok { - l.clusterPolicy = v.(string) + l.ClusterPolicy = v.(string) } if v, ok := m["cluster"]; ok { - l.cluster = v.(string) + l.Cluster = v.(string) } if v, ok := m["dashboard"]; ok { - l.dashboard = v.(string) + l.Dashboard = v.(string) } if v, ok := m["instance-pool"]; ok { - l.instancePool = v.(string) + l.InstancePool = v.(string) } if v, ok := m["job"]; ok { - l.job = v.(string) + l.Job = v.(string) } if v, ok := m["metastore"]; ok { - l.metastore = v.(string) + l.Metastore = v.(string) } if v, ok := m["pipeline"]; ok { - l.pipeline = v.(string) + l.Pipeline = v.(string) } if v, ok := m["query"]; ok { - l.query = v.(string) + l.Query = v.(string) + } + if v, ok := m["service-principal"]; ok { + l.ServicePrincipal = v.(string) } if v, ok := m["warehouse"]; ok { - l.warehouse = v.(string) + l.Warehouse = v.(string) } return l @@ -74,70 +79,76 @@ func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (st } resolvers := resolvers() - if l.alert != "" { - return resolvers["alert"](ctx, w, l.alert) + if l.Alert != "" { + return resolvers["alert"](ctx, w, l.Alert) + } + if l.ClusterPolicy != "" { + return resolvers["cluster-policy"](ctx, w, l.ClusterPolicy) } - if l.clusterPolicy != "" { - return resolvers["cluster-policy"](ctx, w, l.clusterPolicy) + if l.Cluster != "" { + return resolvers["cluster"](ctx, w, l.Cluster) } - if l.cluster != "" { - return resolvers["cluster"](ctx, w, l.cluster) + if l.Dashboard != "" { + return resolvers["dashboard"](ctx, w, l.Dashboard) } - if l.dashboard != "" { - return resolvers["dashboard"](ctx, w, l.dashboard) + if l.InstancePool != "" { + return resolvers["instance-pool"](ctx, w, l.InstancePool) } - if l.instancePool != "" { - return resolvers["instance-pool"](ctx, w, l.instancePool) + if l.Job != "" { + return resolvers["job"](ctx, w, l.Job) } - if l.job != "" { - return resolvers["job"](ctx, w, l.job) + if l.Metastore != "" { + return resolvers["metastore"](ctx, w, l.Metastore) } - if l.metastore != "" { - return resolvers["metastore"](ctx, w, l.metastore) + if l.Pipeline != "" { + return resolvers["pipeline"](ctx, w, l.Pipeline) } - if l.pipeline != "" { - return resolvers["pipeline"](ctx, w, l.pipeline) + if l.Query != "" { + return resolvers["query"](ctx, w, l.Query) } - if l.query != "" { - return resolvers["query"](ctx, w, l.query) + if l.ServicePrincipal != "" { + return resolvers["service-principal"](ctx, w, l.ServicePrincipal) } - if l.warehouse != "" { - return resolvers["warehouse"](ctx, w, l.warehouse) + if l.Warehouse != "" { + return resolvers["warehouse"](ctx, w, l.Warehouse) } return "", fmt.Errorf("no valid lookup fields provided") } func (l *Lookup) String() string { - if l.alert != "" { - return fmt.Sprintf("alert: %s", l.alert) + if l.Alert != "" { + return fmt.Sprintf("alert: %s", l.Alert) } - if l.clusterPolicy != "" { - return fmt.Sprintf("cluster-policy: %s", l.clusterPolicy) + if l.ClusterPolicy != "" { + return fmt.Sprintf("cluster-policy: %s", l.ClusterPolicy) } - if l.cluster != "" { - return fmt.Sprintf("cluster: %s", l.cluster) + if l.Cluster != "" { + return fmt.Sprintf("cluster: %s", l.Cluster) } - if l.dashboard != "" { - return fmt.Sprintf("dashboard: %s", l.dashboard) + if l.Dashboard != "" { + return fmt.Sprintf("dashboard: %s", l.Dashboard) } - if l.instancePool != "" { - return fmt.Sprintf("instance-pool: %s", l.instancePool) + if l.InstancePool != "" { + return fmt.Sprintf("instance-pool: %s", l.InstancePool) } - if l.job != "" { - return fmt.Sprintf("job: %s", l.job) + if l.Job != "" { + return fmt.Sprintf("job: %s", l.Job) } - if l.metastore != "" { - return fmt.Sprintf("metastore: %s", l.metastore) + if l.Metastore != "" { + return fmt.Sprintf("metastore: %s", l.Metastore) } - if l.pipeline != "" { - return fmt.Sprintf("pipeline: %s", l.pipeline) + if l.Pipeline != "" { + return fmt.Sprintf("pipeline: %s", l.Pipeline) } - if l.query != "" { - return fmt.Sprintf("query: %s", l.query) + if l.Query != "" { + return fmt.Sprintf("query: %s", l.Query) } - if l.warehouse != "" { - return fmt.Sprintf("warehouse: %s", l.warehouse) + if l.ServicePrincipal != "" { + return fmt.Sprintf("service-principal: %s", l.ServicePrincipal) + } + if l.Warehouse != "" { + return fmt.Sprintf("warehouse: %s", l.Warehouse) } return "" @@ -146,34 +157,37 @@ func (l *Lookup) String() string { func (l *Lookup) validate() error { // Validate that only one field is set count := 0 - if l.alert != "" { + if l.Alert != "" { + count++ + } + if l.ClusterPolicy != "" { count++ } - if l.clusterPolicy != "" { + if l.Cluster != "" { count++ } - if l.cluster != "" { + if l.Dashboard != "" { count++ } - if l.dashboard != "" { + if l.InstancePool != "" { count++ } - if l.instancePool != "" { + if l.Job != "" { count++ } - if l.job != "" { + if l.Metastore != "" { count++ } - if l.metastore != "" { + if l.Pipeline != "" { count++ } - if l.pipeline != "" { + if l.Query != "" { count++ } - if l.query != "" { + if l.ServicePrincipal != "" { count++ } - if l.warehouse != "" { + if l.Warehouse != "" { count++ } @@ -264,6 +278,14 @@ func resolvers() map[string](resolverFunc) { return fmt.Sprint(entity.Id), nil } + resolvers["service-principal"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { + entity, err := w.ServicePrincipals.GetByDisplayName(ctx, name) + if err != nil { + return "", err + } + + return fmt.Sprint(entity.Id), nil + } resolvers["warehouse"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { entity, err := w.Warehouses.GetByName(ctx, name) if err != nil {