From a8d7a2d933559de68a6501ced6f7b8aca8c915df Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 21 May 2026 14:43:11 -0700 Subject: [PATCH 1/4] Support pre-built function runtime images in projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crossplane projects today discover embedded functions by convention: every subdirectory of paths.functions is treated as a function, and the CLI auto-detects the language and builds the runtime image. This works well for simple projects but blocks projects that have outgrown the built-in builders or that need to coordinate function builds with an existing build system (make, nix, Bazel, CI pipelines). Per https://github.com/crossplane/cli/issues/21, users want to supply pre-built OCI runtime images alongside source-based functions, so the CLI handles packaging while the user owns the build. This commit adds an optional functions list to ProjectSpec. When the list is present it disables auto-discovery and is the sole source of truth for which functions to build. Each entry uses a Source discriminator (Directory or Tarball) and a corresponding sub-field: spec: architectures: [amd64, arm64] functions: - source: Directory directory: name: function-a - source: Tarball tarball: name: function-b pathPrefix: build/function-b Directory-source functions follow the existing build path. Tarball- source functions skip language detection and load one pre-built single-platform OCI image tarball per target architecture, following the naming convention `-.tar`. So the example above loads `build/function-b-amd64.tar` and `build/function-b-arm64.tar`. Per-architecture tarballs match what build tools naturally produce without bundling: `docker save`, Nix's dockerTools.buildImage, Bazel's oci_tarball, `ko build --tarball`, etc. all emit one single-platform tarball at a time. Packaging is inherently per- architecture too — each runtime image gets its own crossplane.yaml layer before they're tied together into a multi-arch package index — so the CLI would have to split a multi-arch input apart anyway. The CLI verifies that each tarball's image config records the architecture its filename promises, and adds the package metadata layer (crossplane.yaml) before assembling the multi-arch package index. The on-disk output is identical to a CLI-built function. When the functions list is omitted, the existing auto-discovery behaviour is preserved unchanged. Fixes https://github.com/crossplane/cli/issues/21. Signed-off-by: Nic Cope --- apis/dev/v1alpha1/project_types.go | 89 ++++++ apis/dev/v1alpha1/validate.go | 96 +++++++ apis/dev/v1alpha1/validate_test.go | 163 +++++++++++ apis/dev/v1alpha1/zz_generated.deepcopy.go | 62 +++++ internal/project/build.go | 212 ++++++++++---- internal/project/build_test.go | 306 +++++++++++++++++++++ 6 files changed, 874 insertions(+), 54 deletions(-) diff --git a/apis/dev/v1alpha1/project_types.go b/apis/dev/v1alpha1/project_types.go index dc9dad53..bf38cab2 100644 --- a/apis/dev/v1alpha1/project_types.go +++ b/apis/dev/v1alpha1/project_types.go @@ -33,6 +33,18 @@ const ( DependencyTypeXpkg = "xpkg" ) +// Function source constants. +const ( + // FunctionSourceDirectory indicates a function whose source code lives in + // a directory under the project's functions path. The CLI builds the + // runtime image from this source. + FunctionSourceDirectory = "Directory" + // FunctionSourceTarball indicates a function whose runtime image is + // supplied as a pre-built OCI image tarball. The CLI skips building and + // uses the tarball as the runtime image. + FunctionSourceTarball = "Tarball" +) + // Project defines a Crossplane Project, which can be built into a Crossplane // Configuration package. // @@ -64,6 +76,12 @@ type ProjectSpec struct { Crossplane *pkgmetav1.CrossplaneConstraints `json:"crossplane,omitempty"` // Dependencies are built-time and runtime dependencies of the project. Dependencies []Dependency `json:"dependencies,omitempty"` + // Functions explicitly declares the embedded functions in this project. + // If specified, automatic discovery of functions under the functions path + // is disabled and only the functions listed here will be built and + // packaged. If omitted, all subdirectories of the functions path are + // treated as Directory-source functions and built automatically. + Functions []Function `json:"functions,omitempty"` // Paths defines the relative paths to various parts of the project. Paths *ProjectPaths `json:"paths,omitempty"` // Architectures indicates for which architectures embedded functions should @@ -186,3 +204,74 @@ type K8sDependency struct { // Version is the Kubernetes API version (e.g., "v1.33.0"). Version string `json:"version"` } + +// Function explicitly declares an embedded function in a Crossplane project. +// The Source field is the discriminator that determines which sub-field is +// relevant. +type Function struct { + // Source defines how the function's runtime image is supplied. + // +kubebuilder:validation:Enum=Directory;Tarball + Source string `json:"source"` + + // Directory describes a function whose source code lives in a directory + // under the project's functions path. The CLI builds the runtime image + // from this source. Only used when Source is "Directory". + // +optional + Directory *FunctionDirectory `json:"directory,omitempty"` + + // Tarball describes a function whose runtime image is supplied as a + // pre-built OCI image tarball. Only used when Source is "Tarball". + // +optional + Tarball *FunctionTarball `json:"tarball,omitempty"` +} + +// Name returns the name of the function, derived from the source-specific +// fields. +func (f *Function) Name() string { + if f == nil { + return "" + } + switch f.Source { + case FunctionSourceDirectory: + if f.Directory == nil { + return "" + } + return f.Directory.Name + case FunctionSourceTarball: + if f.Tarball == nil { + return "" + } + return f.Tarball.Name + } + return "" +} + +// FunctionDirectory describes a function whose source code lives in a +// directory under the project's functions path. +type FunctionDirectory struct { + // Name is the name of the function. It must match the name of a + // subdirectory under the project's functions path. + Name string `json:"name"` +} + +// FunctionTarball describes a function whose runtime images are supplied as +// pre-built single-platform OCI image tarballs (as produced by `docker save`, +// Nix's dockerTools.buildImage, Bazel's oci_tarball, ko --tarball, etc.). +// +// The CLI expects one tarball per target architecture, named according to the +// convention `-.tar` and resolved relative to the project +// root. For example, with PathPrefix "build/function-b" and project +// architectures [amd64, arm64], the CLI loads: +// +// build/function-b-amd64.tar +// build/function-b-arm64.tar +type FunctionTarball struct { + // Name is the name of the function. It is used to derive the OCI + // repository for the function's package as `_`. + Name string `json:"name"` + + // PathPrefix is the prefix of the per-architecture runtime image + // tarballs, relative to the project root. For each target architecture + // the CLI loads the file at `-.tar`. + PathPrefix string `json:"pathPrefix"` +} diff --git a/apis/dev/v1alpha1/validate.go b/apis/dev/v1alpha1/validate.go index 7bb63b20..1d9c6b8f 100644 --- a/apis/dev/v1alpha1/validate.go +++ b/apis/dev/v1alpha1/validate.go @@ -19,6 +19,9 @@ package v1alpha1 import ( "fmt" "path/filepath" + "strings" + + "k8s.io/apimachinery/pkg/util/validation" "github.com/crossplane/crossplane-runtime/v2/pkg/errors" ) @@ -75,6 +78,23 @@ func (s *ProjectSpec) Validate() error { } } + // Validate functions. Names must be unique across the list, regardless of + // source, since the function name is used to derive both the package + // metadata name and the OCI repository. + seen := make(map[string]int, len(s.Functions)) + for i, fn := range s.Functions { + if err := fn.Validate(); err != nil { + errs = append(errs, errors.Wrapf(err, "function %d", i)) + continue + } + name := fn.Name() + if first, ok := seen[name]; ok { + errs = append(errs, errors.Errorf("function %d: name %q is already used by function %d", i, name, first)) + continue + } + seen[name] = i + } + return errors.Join(errs...) } @@ -172,3 +192,79 @@ func (k *K8sDependency) Validate() error { return errors.Join(errs...) } + +// Validate validates a Function declaration. +func (f *Function) Validate() error { + var errs []error + + // Count non-nil sources to enforce that exactly one matches the + // discriminator. + sourceCount := 0 + if f.Directory != nil { + sourceCount++ + } + if f.Tarball != nil { + sourceCount++ + } + if sourceCount != 1 { + errs = append(errs, errors.New("exactly one source (directory or tarball) must be specified")) + } + + switch f.Source { + case FunctionSourceDirectory: + if err := f.Directory.Validate(); err != nil { + errs = append(errs, errors.Wrap(err, "directory")) + } + case FunctionSourceTarball: + if err := f.Tarball.Validate(); err != nil { + errs = append(errs, errors.Wrap(err, "tarball")) + } + case "": + errs = append(errs, errors.New("source must not be empty")) + default: + errs = append(errs, errors.Errorf("source %q is not supported, must be one of %q or %q", f.Source, FunctionSourceDirectory, FunctionSourceTarball)) + } + + return errors.Join(errs...) +} + +// Validate validates a FunctionDirectory. A nil receiver is invalid; this is +// the failure mode when a function is declared with source Directory but no +// directory field set. +func (d *FunctionDirectory) Validate() error { + if d == nil { + return errors.Errorf("source %q requires the directory field to be set", FunctionSourceDirectory) + } + + var errs []error + if d.Name == "" { + errs = append(errs, errors.New("name must not be empty")) + } else if msgs := validation.IsDNS1123Subdomain(d.Name); len(msgs) > 0 { + errs = append(errs, errors.Errorf("name %q is not a valid function name: %s", d.Name, strings.Join(msgs, "; "))) + } + + return errors.Join(errs...) +} + +// Validate validates a FunctionTarball. A nil receiver is invalid; this is +// the failure mode when a function is declared with source Tarball but no +// tarball field set. +func (t *FunctionTarball) Validate() error { + if t == nil { + return errors.Errorf("source %q requires the tarball field to be set", FunctionSourceTarball) + } + + var errs []error + if t.Name == "" { + errs = append(errs, errors.New("name must not be empty")) + } else if msgs := validation.IsDNS1123Subdomain(t.Name); len(msgs) > 0 { + errs = append(errs, errors.Errorf("name %q is not a valid function name: %s", t.Name, strings.Join(msgs, "; "))) + } + if t.PathPrefix == "" { + errs = append(errs, errors.New("pathPrefix must not be empty")) + } else if filepath.IsAbs(t.PathPrefix) { + errs = append(errs, errors.New("pathPrefix must be relative")) + } + + return errors.Join(errs...) +} diff --git a/apis/dev/v1alpha1/validate_test.go b/apis/dev/v1alpha1/validate_test.go index 66c4ab16..6e016535 100644 --- a/apis/dev/v1alpha1/validate_test.go +++ b/apis/dev/v1alpha1/validate_test.go @@ -285,6 +285,169 @@ func TestValidate(t *testing.T) { "dependency 0: k8s: version must not be empty", }, }, + "ValidDirectoryFunction": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceDirectory, + Directory: &FunctionDirectory{Name: "fn-one"}, + }}, + }, + }, + }, + "ValidTarballFunction": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceTarball, + Tarball: &FunctionTarball{Name: "fn-two", PathPrefix: "build/fn-two"}, + }}, + }, + }, + }, + "FunctionMissingSource": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Directory: &FunctionDirectory{Name: "fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + "function 0: source must not be empty", + }, + }, + "FunctionUnknownSource": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: "Mystery", + Directory: &FunctionDirectory{Name: "fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + `function 0: source "Mystery" is not supported`, + }, + }, + "FunctionDirectorySourceMissingDirectory": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceDirectory, + Tarball: &FunctionTarball{Name: "fn-one", PathPrefix: "build/fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + `function 0: directory: source "Directory" requires the directory field to be set`, + }, + }, + "FunctionTarballSourceMissingTarball": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceTarball, + Directory: &FunctionDirectory{Name: "fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + `function 0: tarball: source "Tarball" requires the tarball field to be set`, + }, + }, + "FunctionMultipleSources": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceDirectory, + Directory: &FunctionDirectory{Name: "fn-one"}, + Tarball: &FunctionTarball{Name: "fn-one", PathPrefix: "build/fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + "function 0: exactly one source (directory or tarball) must be specified", + }, + }, + "FunctionDirectoryEmptyName": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceDirectory, + Directory: &FunctionDirectory{}, + }}, + }, + }, + expectedErrors: []string{ + "function 0: directory: name must not be empty", + }, + }, + "FunctionTarballEmptyPathPrefix": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceTarball, + Tarball: &FunctionTarball{Name: "fn-one"}, + }}, + }, + }, + expectedErrors: []string{ + "function 0: tarball: pathPrefix must not be empty", + }, + }, + "FunctionTarballAbsolutePathPrefix": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{{ + Source: FunctionSourceTarball, + Tarball: &FunctionTarball{Name: "fn-one", PathPrefix: "/abs/prefix"}, + }}, + }, + }, + expectedErrors: []string{ + "function 0: tarball: pathPrefix must be relative", + }, + }, + "FunctionDuplicateName": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{Name: "my-project"}, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Functions: []Function{ + {Source: FunctionSourceDirectory, Directory: &FunctionDirectory{Name: "shared"}}, + {Source: FunctionSourceTarball, Tarball: &FunctionTarball{Name: "shared", PathPrefix: "build/shared"}}, + }, + }, + }, + expectedErrors: []string{ + `function 1: name "shared" is already used by function 0`, + }, + }, } for name, tc := range tcs { diff --git a/apis/dev/v1alpha1/zz_generated.deepcopy.go b/apis/dev/v1alpha1/zz_generated.deepcopy.go index 19714fc9..533c8045 100644 --- a/apis/dev/v1alpha1/zz_generated.deepcopy.go +++ b/apis/dev/v1alpha1/zz_generated.deepcopy.go @@ -61,6 +61,61 @@ func (in *Dependency) DeepCopy() *Dependency { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Function) DeepCopyInto(out *Function) { + *out = *in + if in.Directory != nil { + in, out := &in.Directory, &out.Directory + *out = new(FunctionDirectory) + **out = **in + } + if in.Tarball != nil { + in, out := &in.Tarball, &out.Tarball + *out = new(FunctionTarball) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Function. +func (in *Function) DeepCopy() *Function { + if in == nil { + return nil + } + out := new(Function) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FunctionDirectory) DeepCopyInto(out *FunctionDirectory) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FunctionDirectory. +func (in *FunctionDirectory) DeepCopy() *FunctionDirectory { + if in == nil { + return nil + } + out := new(FunctionDirectory) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FunctionTarball) DeepCopyInto(out *FunctionTarball) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FunctionTarball. +func (in *FunctionTarball) DeepCopy() *FunctionTarball { + if in == nil { + return nil + } + out := new(FunctionTarball) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitDependency) DeepCopyInto(out *GitDependency) { *out = *in @@ -178,6 +233,13 @@ func (in *ProjectSpec) DeepCopyInto(out *ProjectSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Functions != nil { + in, out := &in.Functions, &out.Functions + *out = make([]Function, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Paths != nil { in, out := &in.Paths, &out.Paths *out = new(ProjectPaths) diff --git a/internal/project/build.go b/internal/project/build.go index 92628a9e..672ce2bb 100644 --- a/internal/project/build.go +++ b/internal/project/build.go @@ -20,6 +20,7 @@ package project import ( "context" "fmt" + "io" "io/fs" "os" "path/filepath" @@ -29,6 +30,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/spf13/afero" "golang.org/x/sync/errgroup" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -181,6 +183,15 @@ func (b *realBuilder) Build(ctx context.Context, project *devv1alpha1.Project, p } functionsSource := afero.NewBasePathFs(projectFS, project.Spec.Paths.Functions) + + // Determine the set of functions to build. If the project explicitly + // declares a Functions list we use it verbatim. Otherwise we auto-discover + // by listing subdirectories of the functions path. + fns, err := resolveFunctions(project, functionsSource) + if err != nil { + return nil, errors.Wrap(err, "failed to resolve functions") + } + apisSource := projectFS apiExcludes := []string{ project.Spec.Paths.Examples, @@ -248,9 +259,9 @@ func (b *realBuilder) Build(ctx context.Context, project *devv1alpha1.Project, p o.eventCh.SendEvent("Generating schemas", async.EventStatusSuccess) } - // Find and build embedded functions. + // Build the resolved functions. o.log.Debug("Building functions") - imgMap, deps, err := b.buildFunctions(ctx, projectFS, functionsSource, project, o.projectBasePath, o.eventCh) + imgMap, deps, err := b.buildFunctions(ctx, projectFS, functionsSource, project, fns, o.projectBasePath, o.eventCh) if err != nil { return nil, err } @@ -301,50 +312,60 @@ func (b *realBuilder) Build(ctx context.Context, project *devv1alpha1.Project, p return imgMap, nil } -// buildFunctions builds the embedded functions found in directories at the top -// level of the provided filesystem. -func (b *realBuilder) buildFunctions(ctx context.Context, projectFS, fromFS afero.Fs, project *devv1alpha1.Project, basePath string, eventCh async.EventChannel) (ImageTagMap, []xpmetav1.Dependency, error) { - var ( - imgMap = make(map[name.Tag]v1.Image) - imgMu sync.Mutex - ) +// resolveFunctions returns the list of functions to build for the project. If +// the project explicitly declares functions, that list is returned verbatim. +// Otherwise it auto-discovers Directory-source functions by listing +// subdirectories of the project's functions path. +func resolveFunctions(project *devv1alpha1.Project, functionsSource afero.Fs) ([]devv1alpha1.Function, error) { + if len(project.Spec.Functions) > 0 { + return project.Spec.Functions, nil + } - infos, err := afero.ReadDir(fromFS, "/") + infos, err := afero.ReadDir(functionsSource, "/") switch { case os.IsNotExist(err): - return imgMap, nil, nil + return nil, nil case err != nil: - return nil, nil, errors.Wrap(err, "failed to list functions directory") + return nil, errors.Wrap(err, "failed to list functions directory") } - fnDirs := make([]string, 0, len(infos)) + fns := make([]devv1alpha1.Function, 0, len(infos)) for _, info := range infos { - if info.IsDir() { - fnDirs = append(fnDirs, info.Name()) + if !info.IsDir() { + continue } + fns = append(fns, devv1alpha1.Function{ + Source: devv1alpha1.FunctionSourceDirectory, + Directory: &devv1alpha1.FunctionDirectory{Name: info.Name()}, + }) } + return fns, nil +} + +// buildFunctions builds the given list of embedded functions. +func (b *realBuilder) buildFunctions(ctx context.Context, projectFS, fromFS afero.Fs, project *devv1alpha1.Project, fns []devv1alpha1.Function, basePath string, eventCh async.EventChannel) (ImageTagMap, []xpmetav1.Dependency, error) { + var ( + imgMap = make(map[name.Tag]v1.Image) + imgMu sync.Mutex + ) - deps := make([]xpmetav1.Dependency, len(fnDirs)) + deps := make([]xpmetav1.Dependency, len(fns)) eg, ctx := errgroup.WithContext(ctx) sem := make(chan struct{}, b.maxConcurrency) - for i, fnName := range fnDirs { + for i, fn := range fns { eg.Go(func() error { sem <- struct{}{} defer func() { <-sem }() + fnName := fn.Name() eventText := fmt.Sprintf("Building function %s", fnName) eventCh.SendEvent(eventText, async.EventStatusStarted) fnRepo := fmt.Sprintf("%s_%s", project.Spec.Repository, fnName) - fnFS := afero.NewBasePathFs(fromFS, fnName) - fnBasePath := "" - if basePath != "" { - fnBasePath = filepath.Join(basePath, project.Spec.Paths.Functions, fnName) - } - imgs, err := b.buildFunction(ctx, projectFS, fnFS, project, fnName, fnBasePath) + imgs, err := b.buildFunction(ctx, projectFS, fromFS, project, fn, basePath) if err != nil { eventCh.SendEvent(eventText, async.EventStatusFailure) return errors.Wrapf(err, "failed to build function %q", fnName) @@ -387,18 +408,19 @@ func (b *realBuilder) buildFunctions(ctx context.Context, projectFS, fromFS afer }) } - err = eg.Wait() - if err != nil { + if err := eg.Wait(); err != nil { return nil, nil, err } return imgMap, deps, nil } -// buildFunction builds images for a single function whose source resides in the -// given filesystem. -func (b *realBuilder) buildFunction(ctx context.Context, projectFS, fromFS afero.Fs, project *devv1alpha1.Project, fnName string, basePath string) ([]v1.Image, error) { - fn := &xpmetav1.Function{ +// buildFunction builds the package images for a single function. It resolves +// the function's runtime images (either by building from source or by loading +// a pre-built tarball) and then wraps each one with the package metadata. +func (b *realBuilder) buildFunction(ctx context.Context, projectFS, functionsFS afero.Fs, project *devv1alpha1.Project, fn devv1alpha1.Function, basePath string) ([]v1.Image, error) { + fnName := fn.Name() + meta := &xpmetav1.Function{ TypeMeta: metav1.TypeMeta{ APIVersion: xpmetav1.SchemeGroupVersion.String(), Kind: xpmetav1.FunctionKind, @@ -414,7 +436,7 @@ func (b *realBuilder) buildFunction(ctx context.Context, projectFS, fromFS afero }, } metaFS := afero.NewMemMapFs() - y, err := yaml.Marshal(fn) + y, err := yaml.Marshal(meta) if err != nil { return nil, errors.Wrap(err, "failed to marshal function metadata") } @@ -423,18 +445,28 @@ func (b *realBuilder) buildFunction(ctx context.Context, projectFS, fromFS afero return nil, errors.Wrap(err, "failed to write function metadata") } + // Source the examples from the function's own directory if it's a + // Directory-source function. Tarball-source functions don't have a source + // directory under functions/, so they have no examples to ship. examplesParser := parser.NewEchoBackend("") - examplesExist, err := afero.IsDir(fromFS, "/examples") - switch { - case err == nil, os.IsNotExist(err): - default: - return nil, errors.Wrap(err, "failed to check for examples") - } - if examplesExist { - examplesParser = parser.NewFsBackend(fromFS, - parser.FsDir("/examples"), - parser.FsFilters(parser.SkipNotYAML()), - ) + if fn.Source == devv1alpha1.FunctionSourceDirectory { + // Resolve the examples directory relative to functionsFS rather than + // wrapping it in another BasePathFs - it joins the path once here + // instead of on every file operation, and keeps the path visible to + // readers. + examplesDir := filepath.Join(fn.Directory.Name, "examples") + examplesExist, err := afero.IsDir(functionsFS, examplesDir) + switch { + case err == nil, os.IsNotExist(err): + default: + return nil, errors.Wrap(err, "failed to check for examples") + } + if examplesExist { + examplesParser = parser.NewFsBackend(functionsFS, + parser.FsDir(examplesDir), + parser.FsFilters(parser.SkipNotYAML()), + ) + } } pp, err := pyaml.New() @@ -448,37 +480,109 @@ func (b *realBuilder) buildFunction(ctx context.Context, projectFS, fromFS afero examples.New(), ) - fnBuilder, err := b.functionIdentifier.Identify(fromFS, project.Spec.ImageConfigs) + runtimeImages, err := b.runtimeImages(ctx, projectFS, functionsFS, project, fn, basePath) if err != nil { - return nil, errors.Wrap(err, "failed to find a builder") + return nil, err } - if bfs, ok := fromFS.(*afero.BasePathFs); ok && basePath == "" { - basePath = afero.FullBaseFsPath(bfs, ".") + pkgImages := make([]v1.Image, 0, len(runtimeImages)) + for _, img := range runtimeImages { + pkgImage, _, err := builder.Build(ctx, xpkg.WithBase(img)) + if err != nil { + return nil, errors.Wrap(err, "failed to build function package") + } + pkgImages = append(pkgImages, pkgImage) + } + + return pkgImages, nil +} + +// runtimeImages returns the per-architecture runtime images for a function. For +// Directory-source functions this dispatches to the appropriate builder. For +// Tarball-source functions it loads the supplied OCI tarball. +func (b *realBuilder) runtimeImages(ctx context.Context, projectFS, functionsFS afero.Fs, project *devv1alpha1.Project, fn devv1alpha1.Function, basePath string) ([]v1.Image, error) { + switch fn.Source { + case devv1alpha1.FunctionSourceDirectory: + return b.buildDirectoryRuntime(ctx, projectFS, functionsFS, project, fn.Directory, basePath) + case devv1alpha1.FunctionSourceTarball: + return loadTarballRuntime(projectFS, fn.Tarball, project.Spec.Architectures) + default: + // Should be caught at validation time, but be defensive. + return nil, errors.Errorf("unsupported function source %q", fn.Source) + } +} + +// buildDirectoryRuntime invokes the appropriate language builder to produce +// runtime images from a function's source directory. +func (b *realBuilder) buildDirectoryRuntime(ctx context.Context, projectFS, functionsFS afero.Fs, project *devv1alpha1.Project, dir *devv1alpha1.FunctionDirectory, basePath string) ([]v1.Image, error) { + fnFS := afero.NewBasePathFs(functionsFS, dir.Name) + + fnBasePath := "" + if basePath != "" { + fnBasePath = filepath.Join(basePath, project.Spec.Paths.Functions, dir.Name) + } + if bfs, ok := fnFS.(*afero.BasePathFs); ok && fnBasePath == "" { + fnBasePath = afero.FullBaseFsPath(bfs, ".") } - runtimeImages, err := fnBuilder.Build(ctx, functions.BuildContext{ + fnBuilder, err := b.functionIdentifier.Identify(fnFS, project.Spec.ImageConfigs) + if err != nil { + return nil, errors.Wrap(err, "failed to find a builder") + } + + imgs, err := fnBuilder.Build(ctx, functions.BuildContext{ ProjectFS: projectFS, - FunctionPath: filepath.Join(project.Spec.Paths.Functions, fnName), + FunctionPath: filepath.Join(project.Spec.Paths.Functions, dir.Name), SchemasPath: project.Spec.Paths.Schemas, Architectures: project.Spec.Architectures, - OSBasePath: basePath, + OSBasePath: fnBasePath, }) if err != nil { return nil, errors.Wrap(err, "failed to build runtime images") } + return imgs, nil +} - pkgImages := make([]v1.Image, 0, len(runtimeImages)) +// loadTarballRuntime reads one pre-built single-platform OCI image tarball per +// target architecture, following the naming convention +// `-.tar`. The tarball format is the Docker-style image +// tarball produced by `docker save`, Nix's dockerTools.buildImage, Bazel's +// oci_tarball, `ko build --tarball`, etc. +func loadTarballRuntime(projectFS afero.Fs, tb *devv1alpha1.FunctionTarball, architectures []string) ([]v1.Image, error) { + images := make([]v1.Image, 0, len(architectures)) + for _, arch := range architectures { + rel := fmt.Sprintf("%s-%s.tar", tb.PathPrefix, arch) + + img, err := tarball.Image(fsOpener(projectFS, rel), nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to load runtime image for architecture %q from %q", arch, rel) + } - for _, img := range runtimeImages { - pkgImage, _, err := builder.Build(ctx, xpkg.WithBase(img)) + // The image's own config records the platform it was built for. If + // it doesn't match the architecture we expected based on the file + // name, the user has almost certainly made a packaging mistake; + // fail loudly rather than producing a multi-arch index that lies. + cfg, err := img.ConfigFile() if err != nil { - return nil, errors.Wrap(err, "failed to build function package") + return nil, errors.Wrapf(err, "failed to read config for runtime image %q", rel) } - pkgImages = append(pkgImages, pkgImage) + if cfg.Architecture != arch { + return nil, errors.Errorf("runtime image %q reports architecture %q but was expected to be %q", rel, cfg.Architecture, arch) + } + + images = append(images, img) } + return images, nil +} - return pkgImages, nil +// fsOpener returns a tarball.Opener that reads a plain tar file from the given +// filesystem. tarball.Image calls its opener multiple times - once for the +// manifest and once per layer - so each call returns a fresh reader positioned +// at the start of the file. +func fsOpener(fsys afero.Fs, path string) tarball.Opener { + return func() (io.ReadCloser, error) { + return fsys.Open(path) + } } func collectResources(toFS afero.Fs, fromFS afero.Fs, gvks []string, exclude []string) error { diff --git a/internal/project/build_test.go b/internal/project/build_test.go index 8f74c359..eba1f39a 100644 --- a/internal/project/build_test.go +++ b/internal/project/build_test.go @@ -18,10 +18,16 @@ package project import ( "fmt" + "path/filepath" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/spf13/afero" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -296,3 +302,303 @@ func tagsOf(m ImageTagMap) []string { } return out } + +func TestResolveFunctions(t *testing.T) { + t.Parallel() + + tcs := map[string]struct { + spec devv1alpha1.ProjectSpec + fnDirs []string + fnFiles []string // files (not dirs) under the functions path; should be ignored. + want []devv1alpha1.Function + }{ + "ExplicitListWins": { + // When the project declares functions explicitly, + // auto-discovery is disabled and the list is returned verbatim. + spec: devv1alpha1.ProjectSpec{ + Functions: []devv1alpha1.Function{ + {Source: devv1alpha1.FunctionSourceDirectory, Directory: &devv1alpha1.FunctionDirectory{Name: "explicit"}}, + }, + }, + fnDirs: []string{"would-be-discovered"}, + want: []devv1alpha1.Function{ + {Source: devv1alpha1.FunctionSourceDirectory, Directory: &devv1alpha1.FunctionDirectory{Name: "explicit"}}, + }, + }, + "AutoDiscoverDirectories": { + // Every subdirectory of the functions path becomes a + // Directory-source function. + fnDirs: []string{"fn-a", "fn-b"}, + want: []devv1alpha1.Function{ + {Source: devv1alpha1.FunctionSourceDirectory, Directory: &devv1alpha1.FunctionDirectory{Name: "fn-a"}}, + {Source: devv1alpha1.FunctionSourceDirectory, Directory: &devv1alpha1.FunctionDirectory{Name: "fn-b"}}, + }, + }, + "AutoDiscoverIgnoresFiles": { + // Files directly under the functions path are not treated as + // functions; only subdirectories are. + fnDirs: []string{"fn-real"}, + fnFiles: []string{"README.md", "stray.tar"}, + want: []devv1alpha1.Function{ + {Source: devv1alpha1.FunctionSourceDirectory, Directory: &devv1alpha1.FunctionDirectory{Name: "fn-real"}}, + }, + }, + "AutoDiscoverNoFunctionsDir": { + // A missing functions path is not an error; it just yields no + // functions. + }, + } + + for name, tc := range tcs { + t.Run(name, func(t *testing.T) { + t.Parallel() + + projFS := afero.NewMemMapFs() + for _, d := range tc.fnDirs { + if err := projFS.MkdirAll(filepath.Join("functions", d), 0o755); err != nil { + t.Fatal(err) + } + } + for _, f := range tc.fnFiles { + if err := projFS.MkdirAll("functions", 0o755); err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(projFS, filepath.Join("functions", f), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + proj := &devv1alpha1.Project{ + ObjectMeta: metav1.ObjectMeta{Name: "p"}, + Spec: tc.spec, + } + proj.Spec.Repository = "xpkg.crossplane.io/example/test" + proj.Default() + + fnsSource := afero.NewBasePathFs(projFS, proj.Spec.Paths.Functions) + got, err := resolveFunctions(proj, fnsSource) + if err != nil { + t.Fatalf("resolveFunctions: %v", err) + } + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("resolveFunctions(...): -want, +got:\n%s", diff) + } + }) + } +} + +func TestBuilderBuildExplicitFunctions(t *testing.T) { + t.Parallel() + + projFS := afero.NewMemMapFs() + writeProject(t, projFS, + map[string]string{ + "db.yaml": xrdYAML("acme.example.com", "xdatabases", "xdatabase", "XDatabase"), + "db-comp.yaml": compositionYAML("xdb", "acme.example.com", "XDatabase"), + }, + // Auto-discovery would find fn-auto; explicit functions should + // override and only build fn-explicit. + []string{"fn-auto", "fn-explicit"}, + ) + + proj := &devv1alpha1.Project{ + ObjectMeta: metav1.ObjectMeta{Name: "test-project"}, + Spec: devv1alpha1.ProjectSpec{ + Repository: "xpkg.crossplane.io/example/test", + Functions: []devv1alpha1.Function{{ + Source: devv1alpha1.FunctionSourceDirectory, + Directory: &devv1alpha1.FunctionDirectory{Name: "fn-explicit"}, + }}, + }, + } + proj.Default() + + imgMap, err := NewBuilder(BuildWithFunctionIdentifier(functions.FakeIdentifier)).Build(t.Context(), proj, projFS) + if err != nil { + t.Fatalf("Build: %v", err) + } + + // The configuration image plus per-arch images for fn-explicit. fn-auto + // must not appear because the explicit list disables auto-discovery. + want := map[string]bool{ + proj.Spec.Repository: true, + proj.Spec.Repository + "_fn-explicit": true, + } + got := map[string]bool{} + for tag := range imgMap { + got[tag.Repository.Name()] = true + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Build(...) function repos: -want, +got:\n%s", diff) + } +} + +func TestBuilderBuildTarballFunction(t *testing.T) { + t.Parallel() + + // Build one single-platform Docker-style tarball per architecture, named + // using the -.tar convention. + projFS := afero.NewMemMapFs() + for _, arch := range []string{"amd64", "arm64"} { + writeRuntimeTar(t, projFS, "fn-prebuilt-"+arch+".tar", arch) + } + + if err := projFS.MkdirAll("apis", 0o755); err != nil { + t.Fatal(err) + } + if err := afero.WriteFile(projFS, "apis/db.yaml", []byte(xrdYAML("acme.example.com", "xdatabases", "xdatabase", "XDatabase")), 0o644); err != nil { + t.Fatal(err) + } + + proj := &devv1alpha1.Project{ + ObjectMeta: metav1.ObjectMeta{Name: "test-project"}, + Spec: devv1alpha1.ProjectSpec{ + Repository: "xpkg.crossplane.io/example/test", + Architectures: []string{"amd64", "arm64"}, + Functions: []devv1alpha1.Function{{ + Source: devv1alpha1.FunctionSourceTarball, + Tarball: &devv1alpha1.FunctionTarball{Name: "fn-prebuilt", PathPrefix: "fn-prebuilt"}, + }}, + }, + } + proj.Default() + + imgMap, err := NewBuilder(BuildWithFunctionIdentifier(functions.FakeIdentifier)).Build(t.Context(), proj, projFS) + if err != nil { + t.Fatalf("Build: %v", err) + } + + // The pre-built tarballs should produce one package image per target + // architecture under the function's derived repo. + wantRepo := proj.Spec.Repository + "_fn-prebuilt" + want := map[string]int{wantRepo: 2} + got := map[string]int{} + for tag := range imgMap { + got[tag.Repository.Name()]++ + } + if diff := cmp.Diff(want, got, cmpopts.IgnoreMapEntries(func(k string, _ int) bool { + return k != wantRepo + })); diff != "" { + t.Errorf("Build(...) tarball function images: -want, +got:\n%s", diff) + } +} + +func TestLoadTarballRuntime(t *testing.T) { + t.Parallel() + + type args struct { + // files maps relative file names under the project root to the + // architecture the runtime image they contain should report. + files map[string]string + archs []string + } + type want struct { + archs []string + err error + } + + tcs := map[string]struct { + args args + want want + }{ + "AllArchitecturesPresent": { + args: args{ + files: map[string]string{ + "fn-amd64.tar": "amd64", + "fn-arm64.tar": "arm64", + }, + archs: []string{"amd64", "arm64"}, + }, + want: want{archs: []string{"amd64", "arm64"}}, + }, + "MissingArchitectureFile": { + args: args{ + files: map[string]string{ + "fn-amd64.tar": "amd64", + }, + archs: []string{"amd64", "arm64"}, + }, + want: want{err: cmpopts.AnyError}, + }, + "ArchitectureMismatch": { + args: args{ + files: map[string]string{ + "fn-amd64.tar": "arm64", + }, + archs: []string{"amd64"}, + }, + want: want{err: cmpopts.AnyError}, + }, + "SingleArchitecture": { + args: args{ + files: map[string]string{ + "fn-amd64.tar": "amd64", + }, + archs: []string{"amd64"}, + }, + want: want{archs: []string{"amd64"}}, + }, + } + + for name, tc := range tcs { + t.Run(name, func(t *testing.T) { + t.Parallel() + + projFS := afero.NewMemMapFs() + for fname, arch := range tc.args.files { + writeRuntimeTar(t, projFS, fname, arch) + } + + tb := &devv1alpha1.FunctionTarball{Name: "fn", PathPrefix: "fn"} + got, err := loadTarballRuntime(projFS, tb, tc.args.archs) + + if diff := cmp.Diff(tc.want.err, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("loadTarballRuntime(...): -want error, +got error:\n%s", diff) + } + if diff := cmp.Diff(tc.want.archs, archsOf(t, got), cmpopts.EquateEmpty()); diff != "" { + t.Errorf("loadTarballRuntime(...) architectures: -want, +got:\n%s", diff) + } + }) + } +} + +// archsOf returns the architecture each image reports, in order. +func archsOf(t *testing.T, imgs []v1.Image) []string { + t.Helper() + + archs := make([]string, 0, len(imgs)) + for _, img := range imgs { + cfg, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + archs = append(archs, cfg.Architecture) + } + return archs +} + +// writeRuntimeTar writes a single-platform Docker-style image tarball to the +// given path on fsys containing an empty image whose config records the given +// architecture. +func writeRuntimeTar(t *testing.T, fsys afero.Fs, path, arch string) { + t.Helper() + + img, err := mutate.ConfigFile(empty.Image, &v1.ConfigFile{OS: "linux", Architecture: arch}) + if err != nil { + t.Fatal(err) + } + tag, err := name.NewTag("crossplane.io/test:" + arch) + if err != nil { + t.Fatal(err) + } + + f, err := fsys.Create(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + if err := tarball.Write(tag, img, f); err != nil { + t.Fatal(err) + } +} From 389e23b6865e5dc84b8edee0cec33d1c3d7e5283 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 21 May 2026 20:52:21 -0700 Subject: [PATCH 2/4] Support gzipped function runtime image tarballs Nix's dockerTools.buildImage produces gzipped tarballs by default. Some other build tools (Bazel rules_oci's oci_load, certain ko invocations) do the same. With only plain .tar accepted, users of these tools had to add a decompress step to their build pipeline just to feed images to the Crossplane CLI. This commit teaches the function tarball loader to fall back to `-.tar.gz` when `-.tar` is not present, preferring the plain tar when both exist. The gzipped tarball is streamed through gzip.NewReader into go-containerregistry's tarball.Image; no temporary files are written. Signed-off-by: Nic Cope --- apis/dev/v1alpha1/project_types.go | 17 +++--- internal/project/build.go | 91 +++++++++++++++++++++++++++--- internal/project/build_test.go | 48 +++++++++++++++- 3 files changed, 139 insertions(+), 17 deletions(-) diff --git a/apis/dev/v1alpha1/project_types.go b/apis/dev/v1alpha1/project_types.go index bf38cab2..330a4260 100644 --- a/apis/dev/v1alpha1/project_types.go +++ b/apis/dev/v1alpha1/project_types.go @@ -258,13 +258,14 @@ type FunctionDirectory struct { // pre-built single-platform OCI image tarballs (as produced by `docker save`, // Nix's dockerTools.buildImage, Bazel's oci_tarball, ko --tarball, etc.). // -// The CLI expects one tarball per target architecture, named according to the -// convention `-.tar` and resolved relative to the project -// root. For example, with PathPrefix "build/function-b" and project -// architectures [amd64, arm64], the CLI loads: +// The CLI expects one tarball per target architecture, named according to +// the convention `-.tar` or `-.tar.gz`, +// resolved relative to the project root. The CLI prefers the plain `.tar` +// when both are present. For example, with PathPrefix "build/function-b" and +// project architectures [amd64, arm64], the CLI looks for: // -// build/function-b-amd64.tar -// build/function-b-arm64.tar +// build/function-b-amd64.tar (or .tar.gz) +// build/function-b-arm64.tar (or .tar.gz) type FunctionTarball struct { // Name is the name of the function. It is used to derive the OCI // repository for the function's package as `_`. @@ -272,6 +273,8 @@ type FunctionTarball struct { // PathPrefix is the prefix of the per-architecture runtime image // tarballs, relative to the project root. For each target architecture - // the CLI loads the file at `-.tar`. + // the CLI loads either `-.tar` or + // `-.tar.gz`, preferring the former when both are + // present. PathPrefix string `json:"pathPrefix"` } diff --git a/internal/project/build.go b/internal/project/build.go index 672ce2bb..48985cb4 100644 --- a/internal/project/build.go +++ b/internal/project/build.go @@ -18,6 +18,7 @@ limitations under the License. package project import ( + "compress/gzip" "context" "fmt" "io" @@ -544,18 +545,21 @@ func (b *realBuilder) buildDirectoryRuntime(ctx context.Context, projectFS, func } // loadTarballRuntime reads one pre-built single-platform OCI image tarball per -// target architecture, following the naming convention -// `-.tar`. The tarball format is the Docker-style image -// tarball produced by `docker save`, Nix's dockerTools.buildImage, Bazel's -// oci_tarball, `ko build --tarball`, etc. +// target architecture. For each architecture it looks for, in order: +// +// - -.tar +// - -.tar.gz +// +// The tarball format is the Docker-style image tarball produced by +// `docker save`, Nix's dockerTools.buildImage, Bazel's oci_tarball, +// `ko build --tarball`, etc. The gzipped variant is what most Nix image +// builders emit by default. func loadTarballRuntime(projectFS afero.Fs, tb *devv1alpha1.FunctionTarball, architectures []string) ([]v1.Image, error) { images := make([]v1.Image, 0, len(architectures)) for _, arch := range architectures { - rel := fmt.Sprintf("%s-%s.tar", tb.PathPrefix, arch) - - img, err := tarball.Image(fsOpener(projectFS, rel), nil) + img, rel, err := loadRuntimeImage(projectFS, tb.PathPrefix, arch) if err != nil { - return nil, errors.Wrapf(err, "failed to load runtime image for architecture %q from %q", arch, rel) + return nil, err } // The image's own config records the platform it was built for. If @@ -585,6 +589,77 @@ func fsOpener(fsys afero.Fs, path string) tarball.Opener { } } +// loadRuntimeImage loads the runtime image for a single architecture. It tries +// each candidate tarball in turn, preferring the plain .tar over the gzipped +// .tar.gz, and loads the first one that exists. It returns the loaded image and +// the relative path it was loaded from (for error messages). +// +// The tarballs are read through the project filesystem rather than from a real +// on-disk path, so loading works the same whether the project FS is an +// afero.BasePathFs or an in-memory FS in tests. +func loadRuntimeImage(projectFS afero.Fs, prefix, arch string) (v1.Image, string, error) { + candidates := []struct { + path string + opener func(afero.Fs, string) tarball.Opener + }{ + {path: fmt.Sprintf("%s-%s.tar", prefix, arch), opener: fsOpener}, + {path: fmt.Sprintf("%s-%s.tar.gz", prefix, arch), opener: gzipOpener}, + } + + tried := make([]string, 0, len(candidates)) + for _, c := range candidates { + tried = append(tried, c.path) + + exists, err := afero.Exists(projectFS, c.path) + if err != nil { + return nil, c.path, errors.Wrapf(err, "failed to stat runtime image %q", c.path) + } + if !exists { + continue + } + + img, err := tarball.Image(c.opener(projectFS, c.path), nil) + if err != nil { + return nil, c.path, errors.Wrapf(err, "failed to load runtime image for architecture %q from %q", arch, c.path) + } + return img, c.path, nil + } + + return nil, tried[0], errors.Errorf("no runtime image found for architecture %q: looked for %v", arch, tried) +} + +// gzipOpener returns a tarball.Opener that reads a gzipped tar file from the +// given filesystem. Like fsOpener it can be called repeatedly; each call +// returns a fresh decompressing reader that reads the file from the beginning. +func gzipOpener(fsys afero.Fs, path string) tarball.Opener { + return func() (io.ReadCloser, error) { + f, err := fsys.Open(path) + if err != nil { + return nil, err + } + gz, err := gzip.NewReader(f) + if err != nil { + _ = f.Close() + return nil, err + } + return gzipReadCloser{Reader: gz, file: f}, nil + } +} + +// gzipReadCloser ties together a gzip.Reader and the underlying file so that +// closing the gzip reader also closes the file. +type gzipReadCloser struct { + *gzip.Reader + + file afero.File +} + +// Close closes both the gzip reader and the underlying file, joining any errors +// so neither failure is lost. +func (g gzipReadCloser) Close() error { + return errors.Join(g.Reader.Close(), g.file.Close()) +} + func collectResources(toFS afero.Fs, fromFS afero.Fs, gvks []string, exclude []string) error { return afero.Walk(fromFS, "/", func(path string, info fs.FileInfo, err error) error { if err != nil { diff --git a/internal/project/build_test.go b/internal/project/build_test.go index eba1f39a..74b4c7e8 100644 --- a/internal/project/build_test.go +++ b/internal/project/build_test.go @@ -17,8 +17,10 @@ limitations under the License. package project import ( + "compress/gzip" "fmt" "path/filepath" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -538,6 +540,37 @@ func TestLoadTarballRuntime(t *testing.T) { }, want: want{archs: []string{"amd64"}}, }, + "GzippedTarball": { + args: args{ + files: map[string]string{ + "fn-amd64.tar.gz": "amd64", + }, + archs: []string{"amd64"}, + }, + want: want{archs: []string{"amd64"}}, + }, + "MixedPlainAndGzipped": { + args: args{ + files: map[string]string{ + "fn-amd64.tar": "amd64", + "fn-arm64.tar.gz": "arm64", + }, + archs: []string{"amd64", "arm64"}, + }, + want: want{archs: []string{"amd64", "arm64"}}, + }, + "PlainPreferredOverGzipped": { + // When both .tar and .tar.gz exist for the same architecture, + // the plain .tar is used. + args: args{ + files: map[string]string{ + "fn-amd64.tar": "amd64", + "fn-amd64.tar.gz": "arm64", // mismatched on purpose to prove it isn't read. + }, + archs: []string{"amd64"}, + }, + want: want{archs: []string{"amd64"}}, + }, } for name, tc := range tcs { @@ -579,7 +612,7 @@ func archsOf(t *testing.T, imgs []v1.Image) []string { // writeRuntimeTar writes a single-platform Docker-style image tarball to the // given path on fsys containing an empty image whose config records the given -// architecture. +// architecture. If the path ends with ".tar.gz" the tarball is gzipped. func writeRuntimeTar(t *testing.T, fsys afero.Fs, path, arch string) { t.Helper() @@ -598,7 +631,18 @@ func writeRuntimeTar(t *testing.T, fsys afero.Fs, path, arch string) { } defer f.Close() - if err := tarball.Write(tag, img, f); err != nil { + if !strings.HasSuffix(path, ".gz") { + if err := tarball.Write(tag, img, f); err != nil { + t.Fatal(err) + } + return + } + + gz := gzip.NewWriter(f) + if err := tarball.Write(tag, img, gz); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { t.Fatal(err) } } From 38003e5b7ccd1b80d2dfa3516778c61c346351e1 Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Fri, 22 May 2026 12:12:35 -0700 Subject: [PATCH 3/4] Support generating schemas for specific languages By default crossplane project build and crossplane dependency update-cache generate schemas for all four supported languages (Go, JSON, KCL, Python). Per https://github.com/crossplane/cli/issues/29 this is wasteful for projects that only consume some of them: every build generates language bindings the project never imports. This commit adds an optional schemas block to ProjectSpec: spec: schemas: languages: [python] When languages is set, schema generation is restricted to the listed languages. The filter applies both to the project's own XRD schemas and to its declared dependencies, and flows through project build/run and dependency update-cache/clean-cache. When schemas is omitted (the default), all languages are generated as before. The schemas block is nested rather than flat to leave room for future schema-related knobs (output paths, generator-specific options) without scattering schema config across ProjectSpec. The supported language identifiers are defined as constants (SchemaLanguageGo, SchemaLanguageJSON, SchemaLanguageKCL, SchemaLanguagePython) in the API package, with SupportedSchemaLanguages returning the canonical set. The schema generator package consumes these constants directly so the two cannot drift, and a test in the generator package asserts that AllLanguages covers exactly the API's declared set. Fixes https://github.com/crossplane/cli/issues/29. Signed-off-by: Nic Cope --- apis/dev/v1alpha1/project_types.go | 42 +++++++++ apis/dev/v1alpha1/validate.go | 27 ++++++ apis/dev/v1alpha1/validate_test.go | 45 ++++++++++ apis/dev/v1alpha1/zz_generated.deepcopy.go | 25 ++++++ cmd/crossplane/dependency/cache.go | 2 + cmd/crossplane/function/generate.go | 34 +++++++- cmd/crossplane/function/generate_test.go | 53 ++++++++++++ cmd/crossplane/project/build.go | 2 +- cmd/crossplane/project/run.go | 2 +- internal/schemas/generator/go.go | 3 +- internal/schemas/generator/interface.go | 21 ++++- internal/schemas/generator/interface_test.go | 90 ++++++++++++++++++++ internal/schemas/generator/json.go | 3 +- internal/schemas/generator/kcl.go | 3 +- internal/schemas/generator/python.go | 3 +- internal/schemas/manager/manager.go | 3 +- 16 files changed, 349 insertions(+), 9 deletions(-) create mode 100644 internal/schemas/generator/interface_test.go diff --git a/apis/dev/v1alpha1/project_types.go b/apis/dev/v1alpha1/project_types.go index 330a4260..c696e97a 100644 --- a/apis/dev/v1alpha1/project_types.go +++ b/apis/dev/v1alpha1/project_types.go @@ -45,6 +45,27 @@ const ( FunctionSourceTarball = "Tarball" ) +// Schema language constants. These are the values accepted in +// ProjectSchemas.Languages. Each corresponds to a schema generator in +// internal/schemas/generator. +const ( + SchemaLanguageGo = "go" + SchemaLanguageJSON = "json" + SchemaLanguageKCL = "kcl" + SchemaLanguagePython = "python" +) + +// SupportedSchemaLanguages returns the set of language identifiers accepted +// in ProjectSchemas.Languages. +func SupportedSchemaLanguages() []string { + return []string{ + SchemaLanguageGo, + SchemaLanguageJSON, + SchemaLanguageKCL, + SchemaLanguagePython, + } +} + // Project defines a Crossplane Project, which can be built into a Crossplane // Configuration package. // @@ -87,6 +108,9 @@ type ProjectSpec struct { // Architectures indicates for which architectures embedded functions should // be built. If not specified, it defaults to [amd64, arm64]. Architectures []string `json:"architectures,omitempty"` + // Schemas configures language-specific schema generation for the + // project's XRDs and declared dependencies. + Schemas *ProjectSchemas `json:"schemas,omitempty"` // ImageConfigs configure how images are fetched during // development. Currently, only rewriting is supported; other options will // be silently ignored. Note that these configs are for development only; @@ -105,6 +129,24 @@ type ProjectPackageMetadata struct { Readme string `json:"readme,omitempty"` } +// ProjectSchemas configures language-specific schema generation. Schemas are +// produced both for the project's own XRDs and for its declared dependencies. +type ProjectSchemas struct { + // Languages restricts schema generation to the listed languages. + // Supported values are "go", "json", "kcl", and "python". If not + // specified, schemas are generated for all supported languages. + Languages []string `json:"languages,omitempty"` +} + +// GetLanguages returns the configured schema languages, or nil if no Schemas +// config is set. It is safe to call on a nil receiver. +func (s *ProjectSchemas) GetLanguages() []string { + if s == nil { + return nil + } + return s.Languages +} + // ProjectPaths configures the locations of various parts of the project, for // use at build time. All paths must be relative to the project root. type ProjectPaths struct { diff --git a/apis/dev/v1alpha1/validate.go b/apis/dev/v1alpha1/validate.go index 1d9c6b8f..fe40a985 100644 --- a/apis/dev/v1alpha1/validate.go +++ b/apis/dev/v1alpha1/validate.go @@ -19,6 +19,7 @@ package v1alpha1 import ( "fmt" "path/filepath" + "slices" "strings" "k8s.io/apimachinery/pkg/util/validation" @@ -71,6 +72,8 @@ func (s *ProjectSpec) Validate() error { errs = append(errs, errors.New("architectures must not be empty")) } + errs = append(errs, s.Schemas.Validate()...) + // Validate dependencies for i, dep := range s.Dependencies { if err := dep.Validate(); err != nil { @@ -98,6 +101,30 @@ func (s *ProjectSpec) Validate() error { return errors.Join(errs...) } +// Validate returns errors for an invalid ProjectSchemas. A nil receiver is +// valid (it means "generate schemas for all languages"); an explicitly empty +// Languages list is rejected because it would disable all schema generation, +// which is almost certainly a mistake. +func (s *ProjectSchemas) Validate() []error { + if s == nil { + return nil + } + if s.Languages == nil { + return nil + } + if len(s.Languages) == 0 { + return []error{errors.New("schemas.languages must not be empty when specified")} + } + supported := SupportedSchemaLanguages() + var errs []error + for i, lang := range s.Languages { + if !slices.Contains(supported, lang) { + errs = append(errs, errors.Errorf("schemas.languages[%d]: %q is not a supported schema language; set it to one of %v", i, lang, supported)) + } + } + return errs +} + // Validate validates a dependency. func (d *Dependency) Validate() error { var errs []error diff --git a/apis/dev/v1alpha1/validate_test.go b/apis/dev/v1alpha1/validate_test.go index 6e016535..d7457c56 100644 --- a/apis/dev/v1alpha1/validate_test.go +++ b/apis/dev/v1alpha1/validate_test.go @@ -140,6 +140,51 @@ func TestValidate(t *testing.T) { "architectures must not be empty", }, }, + "ValidSchemaLanguages": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Schemas: &ProjectSchemas{ + Languages: []string{"python"}, + }, + }, + }, + }, + "EmptySchemaLanguages": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Schemas: &ProjectSchemas{ + Languages: []string{}, + }, + }, + }, + expectedErrors: []string{ + "schemas.languages must not be empty when specified", + }, + }, + "UnsupportedSchemaLanguage": { + input: &Project{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-project", + }, + Spec: ProjectSpec{ + Repository: "xpkg.upbound.io/acmeco/my-project", + Schemas: &ProjectSchemas{ + Languages: []string{"python", "fortran"}, + }, + }, + }, + expectedErrors: []string{ + `schemas.languages[1]: "fortran" is not a supported schema language`, + }, + }, "ValidAPIDependency": { input: &Project{ ObjectMeta: metav1.ObjectMeta{ diff --git a/apis/dev/v1alpha1/zz_generated.deepcopy.go b/apis/dev/v1alpha1/zz_generated.deepcopy.go index 533c8045..dc5d5532 100644 --- a/apis/dev/v1alpha1/zz_generated.deepcopy.go +++ b/apis/dev/v1alpha1/zz_generated.deepcopy.go @@ -217,6 +217,26 @@ func (in *ProjectPaths) DeepCopy() *ProjectPaths { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProjectSchemas) DeepCopyInto(out *ProjectSchemas) { + *out = *in + if in.Languages != nil { + in, out := &in.Languages, &out.Languages + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProjectSchemas. +func (in *ProjectSchemas) DeepCopy() *ProjectSchemas { + if in == nil { + return nil + } + out := new(ProjectSchemas) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProjectSpec) DeepCopyInto(out *ProjectSpec) { *out = *in @@ -250,6 +270,11 @@ func (in *ProjectSpec) DeepCopyInto(out *ProjectSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Schemas != nil { + in, out := &in.Schemas, &out.Schemas + *out = new(ProjectSchemas) + (*in).DeepCopyInto(*out) + } if in.ImageConfigs != nil { in, out := &in.ImageConfigs, &out.ImageConfigs *out = make([]v1beta1.ImageConfig, len(*in)) diff --git a/cmd/crossplane/dependency/cache.go b/cmd/crossplane/dependency/cache.go index 81d3804e..16a2bd43 100644 --- a/cmd/crossplane/dependency/cache.go +++ b/cmd/crossplane/dependency/cache.go @@ -29,6 +29,7 @@ import ( "github.com/crossplane/cli/v2/internal/async" "github.com/crossplane/cli/v2/internal/dependency" "github.com/crossplane/cli/v2/internal/project/projectfile" + "github.com/crossplane/cli/v2/internal/schemas/generator" "github.com/crossplane/cli/v2/internal/terminal" clixpkg "github.com/crossplane/cli/v2/internal/xpkg" @@ -83,6 +84,7 @@ func (c *updateCacheCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter) opts := []dependency.ManagerOption{ dependency.WithProjectFile(c.ProjectFile), + dependency.WithSchemaGenerators(generator.Filter(generator.AllLanguages(), proj.Spec.Schemas.GetLanguages())), dependency.WithXpkgClient(client), dependency.WithResolver(resolver), } diff --git a/cmd/crossplane/function/generate.go b/cmd/crossplane/function/generate.go index 3c34fe54..52c57734 100644 --- a/cmd/crossplane/function/generate.go +++ b/cmd/crossplane/function/generate.go @@ -25,6 +25,7 @@ import ( "io/fs" "path" "path/filepath" + "slices" "strings" "text/template" @@ -103,6 +104,10 @@ func (c *generateCmd) AfterApply() error { } c.proj = proj + if err := validateLanguageAgainstSchemas(c.Language, proj.Spec.Schemas.GetLanguages()); err != nil { + return err + } + c.functionsFS = afero.NewBasePathFs(c.projFS, proj.Spec.Paths.Functions) c.schemasFS = afero.NewBasePathFs(c.projFS, proj.Spec.Paths.Schemas) c.fsPath = path.Join(proj.Spec.Paths.Functions, c.Name) @@ -111,6 +116,33 @@ func (c *generateCmd) AfterApply() error { return nil } +// validateLanguageAgainstSchemas refuses to generate a function in a language +// whose schemas the project doesn't generate. Such a function would have no +// models to import, which is surprising, so we fail up front rather than +// scaffolding a function that can't compile. An empty schemaLangs means the +// project generates all languages (matching generator.Filter), so any function +// language is fine. +func validateLanguageAgainstSchemas(functionLang string, schemaLangs []string) error { + if len(schemaLangs) == 0 { + return nil + } + required := functionSchemaLanguage(functionLang) + if !slices.Contains(schemaLangs, required) { + return errors.Errorf("cannot generate a %q function: the project only generates %v schemas; add %q to spec.schemas.languages or choose a different language", functionLang, schemaLangs, required) + } + return nil +} + +// functionSchemaLanguage returns the schema language a generated function in +// the given function language consumes. Most function languages map to a +// like-named schema language; go-templating consumes the JSON schema. +func functionSchemaLanguage(functionLang string) string { + if functionLang == "go-templating" { + return v1alpha1.SchemaLanguageJSON + } + return functionLang +} + // Run generates a function scaffold. func (c *generateCmd) Run(sp terminal.SpinnerPrinter) error { if err := c.validatePaths(); err != nil { @@ -124,7 +156,7 @@ func (c *generateCmd) Run(sp terminal.SpinnerPrinter) error { } schemaMgr := manager.New( c.schemasFS, - generator.AllLanguages(), + generator.Filter(generator.AllLanguages(), c.proj.Spec.Schemas.GetLanguages()), runner.NewRealSchemaRunner(runner.WithImageConfig(c.proj.Spec.ImageConfigs)), ) diff --git a/cmd/crossplane/function/generate_test.go b/cmd/crossplane/function/generate_test.go index eee13725..d23dade4 100644 --- a/cmd/crossplane/function/generate_test.go +++ b/cmd/crossplane/function/generate_test.go @@ -336,6 +336,59 @@ func TestRunErrors(t *testing.T) { } } +func TestValidateLanguageAgainstSchemas(t *testing.T) { + cases := map[string]struct { + functionLang string + schemaLangs []string + wantErrSubstring string + }{ + "NoSchemaRestriction": { + functionLang: "python", + schemaLangs: nil, + }, + "LanguageAllowed": { + functionLang: "python", + schemaLangs: []string{"python"}, + }, + "GoTemplatingMapsToJSON": { + functionLang: "go-templating", + schemaLangs: []string{"json"}, + }, + "GoMapsToGo": { + functionLang: "go", + schemaLangs: []string{"go"}, + }, + "LanguageExcluded": { + functionLang: "python", + schemaLangs: []string{"go"}, + wantErrSubstring: "the project only generates", + }, + "GoTemplatingExcludedWhenOnlyGo": { + functionLang: "go-templating", + schemaLangs: []string{"go"}, + wantErrSubstring: `add "json"`, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + err := validateLanguageAgainstSchemas(tc.functionLang, tc.schemaLangs) + if tc.wantErrSubstring == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil { + t.Fatalf("expected error containing %q, got nil", tc.wantErrSubstring) + } + if !strings.Contains(err.Error(), tc.wantErrSubstring) { + t.Errorf("error = %q, want substring %q", err.Error(), tc.wantErrSubstring) + } + }) + } +} + func TestAddCompositionStep(t *testing.T) { cases := map[string]struct { start []apiextv1.PipelineStep diff --git a/cmd/crossplane/project/build.go b/cmd/crossplane/project/build.go index aa400a1a..ccb4fd17 100644 --- a/cmd/crossplane/project/build.go +++ b/cmd/crossplane/project/build.go @@ -96,7 +96,7 @@ func (c *buildCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter) error concurrency := max(1, c.MaxConcurrency) schemasFS := afero.NewBasePathFs(c.projFS, c.proj.Spec.Paths.Schemas) - generators := generator.AllLanguages() + generators := generator.Filter(generator.AllLanguages(), c.proj.Spec.Schemas.GetLanguages()) schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(c.proj.Spec.ImageConfigs)) schemaMgr := manager.New(schemasFS, generators, schemaRunner) cacheDir := c.CacheDir diff --git a/cmd/crossplane/project/run.go b/cmd/crossplane/project/run.go index 2bf2728c..8ecc5a64 100644 --- a/cmd/crossplane/project/run.go +++ b/cmd/crossplane/project/run.go @@ -149,7 +149,7 @@ func (c *runCmd) Run(logger logging.Logger, sp terminal.SpinnerPrinter) error { concurrency := max(1, c.MaxConcurrency) schemasFS := afero.NewBasePathFs(c.projFS, c.proj.Spec.Paths.Schemas) - generators := generator.AllLanguages() + generators := generator.Filter(generator.AllLanguages(), c.proj.Spec.Schemas.GetLanguages()) schemaRunner := runner.NewRealSchemaRunner(runner.WithImageConfig(c.proj.Spec.ImageConfigs)) schemaMgr := manager.New(schemasFS, generators, schemaRunner) cacheDir := c.CacheDir diff --git a/internal/schemas/generator/go.go b/internal/schemas/generator/go.go index 9e58f902..b83f58cc 100644 --- a/internal/schemas/generator/go.go +++ b/internal/schemas/generator/go.go @@ -47,6 +47,7 @@ import ( xpv1 "github.com/crossplane/crossplane/apis/v2/apiextensions/v1" + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/crd" "github.com/crossplane/cli/v2/internal/schemas/runner" ) @@ -140,7 +141,7 @@ var ( type goGenerator struct{} func (goGenerator) Language() string { - return "go" + return devv1alpha1.SchemaLanguageGo } // GenerateFromCRD generates Go schemas for the CRDs in the given filesystem. diff --git a/internal/schemas/generator/interface.go b/internal/schemas/generator/interface.go index 06ef60f6..d41300b9 100644 --- a/internal/schemas/generator/interface.go +++ b/internal/schemas/generator/interface.go @@ -20,6 +20,7 @@ package generator import ( "context" + "slices" "github.com/spf13/afero" @@ -33,7 +34,9 @@ type Interface interface { GenerateFromOpenAPI(ctx context.Context, fs afero.Fs, runner runner.SchemaRunner) (afero.Fs, error) } -// AllLanguages returns generators for all supported languages. +// AllLanguages returns generators for all supported languages. The set of +// supported language identifiers is defined by +// devv1alpha1.SupportedSchemaLanguages. func AllLanguages() []Interface { return []Interface{ &goGenerator{}, @@ -42,3 +45,19 @@ func AllLanguages() []Interface { &pythonGenerator{}, } } + +// Filter returns the subset of generators whose language identifier appears +// in langs. The order of generators in the result matches the order of all. +// If langs is empty, all generators are returned unchanged. +func Filter(all []Interface, langs []string) []Interface { + if len(langs) == 0 { + return all + } + out := make([]Interface, 0, len(all)) + for _, g := range all { + if slices.Contains(langs, g.Language()) { + out = append(out, g) + } + } + return out +} diff --git a/internal/schemas/generator/interface_test.go b/internal/schemas/generator/interface_test.go new file mode 100644 index 00000000..adf864f6 --- /dev/null +++ b/internal/schemas/generator/interface_test.go @@ -0,0 +1,90 @@ +/* +Copyright 2026 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" +) + +func TestAllLanguagesMatchesAPI(t *testing.T) { + t.Parallel() + + // The generators returned by AllLanguages must cover exactly the set + // of language identifiers declared in the API package. If this test + // fails the two are out of sync; update one to match the other. + got := make([]string, 0, len(AllLanguages())) + for _, g := range AllLanguages() { + got = append(got, g.Language()) + } + if diff := cmp.Diff(devv1alpha1.SupportedSchemaLanguages(), got); diff != "" { + t.Errorf("AllLanguages() languages: -want (from API), +got (from generators):\n%s", diff) + } +} + +func TestFilter(t *testing.T) { + t.Parallel() + + all := AllLanguages() + + tcs := map[string]struct { + langs []string + want []string + }{ + "Empty": { + // An empty filter returns all languages unchanged. + want: devv1alpha1.SupportedSchemaLanguages(), + }, + "SingleLanguage": { + langs: []string{devv1alpha1.SchemaLanguagePython}, + want: []string{devv1alpha1.SchemaLanguagePython}, + }, + "PreservesAllLanguagesOrder": { + // Filter preserves the order of AllLanguages, not the order + // of the input list. + langs: []string{devv1alpha1.SchemaLanguagePython, devv1alpha1.SchemaLanguageGo}, + want: []string{devv1alpha1.SchemaLanguageGo, devv1alpha1.SchemaLanguagePython}, + }, + "UnknownLanguageIgnored": { + // Filter is permissive; validation happens elsewhere. + langs: []string{devv1alpha1.SchemaLanguagePython, "fortran"}, + want: []string{devv1alpha1.SchemaLanguagePython}, + }, + "AllLanguages": { + langs: devv1alpha1.SupportedSchemaLanguages(), + want: devv1alpha1.SupportedSchemaLanguages(), + }, + } + + for name, tc := range tcs { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got := Filter(all, tc.langs) + gotLangs := make([]string, len(got)) + for i, g := range got { + gotLangs[i] = g.Language() + } + if diff := cmp.Diff(tc.want, gotLangs); diff != "" { + t.Errorf("Filter(...): -want, +got:\n%s", diff) + } + }) + } +} diff --git a/internal/schemas/generator/json.go b/internal/schemas/generator/json.go index cee0d073..8ddc45a3 100644 --- a/internal/schemas/generator/json.go +++ b/internal/schemas/generator/json.go @@ -31,13 +31,14 @@ import ( "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/schemas/runner" ) type jsonGenerator struct{} func (jsonGenerator) Language() string { - return "json" + return devv1alpha1.SchemaLanguageJSON } // GenerateFromCRD generates jsonschemas for the CRDs in the given filesystem. diff --git a/internal/schemas/generator/kcl.go b/internal/schemas/generator/kcl.go index 4aa69db1..0a3e9cff 100644 --- a/internal/schemas/generator/kcl.go +++ b/internal/schemas/generator/kcl.go @@ -42,6 +42,7 @@ import ( xpv1 "github.com/crossplane/crossplane/apis/v2/apiextensions/v1" + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" xcrd "github.com/crossplane/cli/v2/internal/crd" "github.com/crossplane/cli/v2/internal/filesystem" "github.com/crossplane/cli/v2/internal/schemas/runner" @@ -56,7 +57,7 @@ const ( type kclGenerator struct{} func (kclGenerator) Language() string { - return "kcl" + return devv1alpha1.SchemaLanguageKCL } // GenerateFromCRD generates KCL schema files from the XRDs and CRDs fromFS. diff --git a/internal/schemas/generator/python.go b/internal/schemas/generator/python.go index cf7a043e..b6e84617 100644 --- a/internal/schemas/generator/python.go +++ b/internal/schemas/generator/python.go @@ -37,6 +37,7 @@ import ( xpv1 "github.com/crossplane/crossplane/apis/v2/apiextensions/v1" + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/crd" "github.com/crossplane/cli/v2/internal/filesystem" "github.com/crossplane/cli/v2/internal/schemas/runner" @@ -55,7 +56,7 @@ var importRE = regexp.MustCompile(`^(from\s+)(\.*)([^\s]+)(.*)`) type pythonGenerator struct{} func (pythonGenerator) Language() string { - return "python" + return devv1alpha1.SchemaLanguagePython } // GenerateFromCRD generates Python schema files from the XRDs and CRDs fromFS. diff --git a/internal/schemas/manager/manager.go b/internal/schemas/manager/manager.go index ef975e73..fd35fe29 100644 --- a/internal/schemas/manager/manager.go +++ b/internal/schemas/manager/manager.go @@ -30,6 +30,7 @@ import ( "github.com/crossplane/crossplane-runtime/v2/pkg/errors" + devv1alpha1 "github.com/crossplane/cli/v2/apis/dev/v1alpha1" "github.com/crossplane/cli/v2/internal/filesystem" "github.com/crossplane/cli/v2/internal/schemas/generator" "github.com/crossplane/cli/v2/internal/schemas/runner" @@ -143,7 +144,7 @@ func (m *Manager) Generate(ctx context.Context, source Source) (map[string]afero func postProcessForLanguage(language string, langFS afero.Fs) error { switch language { - case "json": + case devv1alpha1.SchemaLanguageJSON: if err := jsonBuildIndexSchema(langFS); err != nil { return errors.Wrap(err, "failed to build index schema for JSON") } From 0a780a5c3d4d81c84b4ed77f060e66b36b72889c Mon Sep 17 00:00:00 2001 From: Nic Cope Date: Thu, 4 Jun 2026 16:56:19 -0700 Subject: [PATCH 4/4] Build embedded functions relative to a single project filesystem The function build path threaded two filesystems through its call chain: the project root, and a separate afero.BasePathFs rooted at the project's functions directory. Both pointed at the same underlying tree, so most function operations were addressed relative to the functions directory while runtime tarballs and the language builder's ProjectFS were addressed relative to the project root. Carrying both meant every function in the chain took two afero.Fs arguments, and the per-function builder filesystem was a BasePathFs wrapped over another BasePathFs. This change drops the functions-rooted filesystem and addresses everything relative to the project root, joining spec.paths.functions inline where the functions directory was previously the root. The build chain now takes a single afero.Fs, and the per-function builder filesystem wraps the project filesystem once instead of twice. Signed-off-by: Nic Cope --- internal/project/build.go | 36 ++++++++++++++-------------------- internal/project/build_test.go | 3 +-- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/internal/project/build.go b/internal/project/build.go index 48985cb4..de487899 100644 --- a/internal/project/build.go +++ b/internal/project/build.go @@ -183,12 +183,10 @@ func (b *realBuilder) Build(ctx context.Context, project *devv1alpha1.Project, p } } - functionsSource := afero.NewBasePathFs(projectFS, project.Spec.Paths.Functions) - // Determine the set of functions to build. If the project explicitly // declares a Functions list we use it verbatim. Otherwise we auto-discover // by listing subdirectories of the functions path. - fns, err := resolveFunctions(project, functionsSource) + fns, err := resolveFunctions(project, projectFS) if err != nil { return nil, errors.Wrap(err, "failed to resolve functions") } @@ -262,7 +260,7 @@ func (b *realBuilder) Build(ctx context.Context, project *devv1alpha1.Project, p // Build the resolved functions. o.log.Debug("Building functions") - imgMap, deps, err := b.buildFunctions(ctx, projectFS, functionsSource, project, fns, o.projectBasePath, o.eventCh) + imgMap, deps, err := b.buildFunctions(ctx, projectFS, project, fns, o.projectBasePath, o.eventCh) if err != nil { return nil, err } @@ -317,12 +315,12 @@ func (b *realBuilder) Build(ctx context.Context, project *devv1alpha1.Project, p // the project explicitly declares functions, that list is returned verbatim. // Otherwise it auto-discovers Directory-source functions by listing // subdirectories of the project's functions path. -func resolveFunctions(project *devv1alpha1.Project, functionsSource afero.Fs) ([]devv1alpha1.Function, error) { +func resolveFunctions(project *devv1alpha1.Project, projectFS afero.Fs) ([]devv1alpha1.Function, error) { if len(project.Spec.Functions) > 0 { return project.Spec.Functions, nil } - infos, err := afero.ReadDir(functionsSource, "/") + infos, err := afero.ReadDir(projectFS, project.Spec.Paths.Functions) switch { case os.IsNotExist(err): return nil, nil @@ -344,7 +342,7 @@ func resolveFunctions(project *devv1alpha1.Project, functionsSource afero.Fs) ([ } // buildFunctions builds the given list of embedded functions. -func (b *realBuilder) buildFunctions(ctx context.Context, projectFS, fromFS afero.Fs, project *devv1alpha1.Project, fns []devv1alpha1.Function, basePath string, eventCh async.EventChannel) (ImageTagMap, []xpmetav1.Dependency, error) { +func (b *realBuilder) buildFunctions(ctx context.Context, projectFS afero.Fs, project *devv1alpha1.Project, fns []devv1alpha1.Function, basePath string, eventCh async.EventChannel) (ImageTagMap, []xpmetav1.Dependency, error) { var ( imgMap = make(map[name.Tag]v1.Image) imgMu sync.Mutex @@ -366,7 +364,7 @@ func (b *realBuilder) buildFunctions(ctx context.Context, projectFS, fromFS afer eventCh.SendEvent(eventText, async.EventStatusStarted) fnRepo := fmt.Sprintf("%s_%s", project.Spec.Repository, fnName) - imgs, err := b.buildFunction(ctx, projectFS, fromFS, project, fn, basePath) + imgs, err := b.buildFunction(ctx, projectFS, project, fn, basePath) if err != nil { eventCh.SendEvent(eventText, async.EventStatusFailure) return errors.Wrapf(err, "failed to build function %q", fnName) @@ -419,7 +417,7 @@ func (b *realBuilder) buildFunctions(ctx context.Context, projectFS, fromFS afer // buildFunction builds the package images for a single function. It resolves // the function's runtime images (either by building from source or by loading // a pre-built tarball) and then wraps each one with the package metadata. -func (b *realBuilder) buildFunction(ctx context.Context, projectFS, functionsFS afero.Fs, project *devv1alpha1.Project, fn devv1alpha1.Function, basePath string) ([]v1.Image, error) { +func (b *realBuilder) buildFunction(ctx context.Context, projectFS afero.Fs, project *devv1alpha1.Project, fn devv1alpha1.Function, basePath string) ([]v1.Image, error) { fnName := fn.Name() meta := &xpmetav1.Function{ TypeMeta: metav1.TypeMeta{ @@ -451,19 +449,15 @@ func (b *realBuilder) buildFunction(ctx context.Context, projectFS, functionsFS // directory under functions/, so they have no examples to ship. examplesParser := parser.NewEchoBackend("") if fn.Source == devv1alpha1.FunctionSourceDirectory { - // Resolve the examples directory relative to functionsFS rather than - // wrapping it in another BasePathFs - it joins the path once here - // instead of on every file operation, and keeps the path visible to - // readers. - examplesDir := filepath.Join(fn.Directory.Name, "examples") - examplesExist, err := afero.IsDir(functionsFS, examplesDir) + examplesDir := filepath.Join(project.Spec.Paths.Functions, fn.Directory.Name, "examples") + examplesExist, err := afero.IsDir(projectFS, examplesDir) switch { case err == nil, os.IsNotExist(err): default: return nil, errors.Wrap(err, "failed to check for examples") } if examplesExist { - examplesParser = parser.NewFsBackend(functionsFS, + examplesParser = parser.NewFsBackend(projectFS, parser.FsDir(examplesDir), parser.FsFilters(parser.SkipNotYAML()), ) @@ -481,7 +475,7 @@ func (b *realBuilder) buildFunction(ctx context.Context, projectFS, functionsFS examples.New(), ) - runtimeImages, err := b.runtimeImages(ctx, projectFS, functionsFS, project, fn, basePath) + runtimeImages, err := b.runtimeImages(ctx, projectFS, project, fn, basePath) if err != nil { return nil, err } @@ -501,10 +495,10 @@ func (b *realBuilder) buildFunction(ctx context.Context, projectFS, functionsFS // runtimeImages returns the per-architecture runtime images for a function. For // Directory-source functions this dispatches to the appropriate builder. For // Tarball-source functions it loads the supplied OCI tarball. -func (b *realBuilder) runtimeImages(ctx context.Context, projectFS, functionsFS afero.Fs, project *devv1alpha1.Project, fn devv1alpha1.Function, basePath string) ([]v1.Image, error) { +func (b *realBuilder) runtimeImages(ctx context.Context, projectFS afero.Fs, project *devv1alpha1.Project, fn devv1alpha1.Function, basePath string) ([]v1.Image, error) { switch fn.Source { case devv1alpha1.FunctionSourceDirectory: - return b.buildDirectoryRuntime(ctx, projectFS, functionsFS, project, fn.Directory, basePath) + return b.buildDirectoryRuntime(ctx, projectFS, project, fn.Directory, basePath) case devv1alpha1.FunctionSourceTarball: return loadTarballRuntime(projectFS, fn.Tarball, project.Spec.Architectures) default: @@ -515,8 +509,8 @@ func (b *realBuilder) runtimeImages(ctx context.Context, projectFS, functionsFS // buildDirectoryRuntime invokes the appropriate language builder to produce // runtime images from a function's source directory. -func (b *realBuilder) buildDirectoryRuntime(ctx context.Context, projectFS, functionsFS afero.Fs, project *devv1alpha1.Project, dir *devv1alpha1.FunctionDirectory, basePath string) ([]v1.Image, error) { - fnFS := afero.NewBasePathFs(functionsFS, dir.Name) +func (b *realBuilder) buildDirectoryRuntime(ctx context.Context, projectFS afero.Fs, project *devv1alpha1.Project, dir *devv1alpha1.FunctionDirectory, basePath string) ([]v1.Image, error) { + fnFS := afero.NewBasePathFs(projectFS, filepath.Join(project.Spec.Paths.Functions, dir.Name)) fnBasePath := "" if basePath != "" { diff --git a/internal/project/build_test.go b/internal/project/build_test.go index 74b4c7e8..7cdbd975 100644 --- a/internal/project/build_test.go +++ b/internal/project/build_test.go @@ -377,8 +377,7 @@ func TestResolveFunctions(t *testing.T) { proj.Spec.Repository = "xpkg.crossplane.io/example/test" proj.Default() - fnsSource := afero.NewBasePathFs(projFS, proj.Spec.Paths.Functions) - got, err := resolveFunctions(proj, fnsSource) + got, err := resolveFunctions(proj, projFS) if err != nil { t.Fatalf("resolveFunctions: %v", err) }