feat(feature): consume collection.json from OCI feature registries

pkg/devcontainer/feature/collection.go

@@ -0,0 +1,137 @@
+package feature
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/devsy-org/devsy/pkg/image"
+	"github.com/devsy-org/devsy/pkg/log"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+)
+
+const CollectionLayerMediaType = "application/vnd.devcontainers.collection.layer.v1+json"
+
+type CollectionFeature struct {
+	ID               string         `json:"id"`
+	Version          string         `json:"version"`
+	Name             string         `json:"name"`
+	Description      string         `json:"description"`
+	DocumentationURL string         `json:"documentationURL,omitempty"`
+	Options          map[string]any `json:"options,omitempty"`
+	Deprecated       bool           `json:"deprecated,omitempty"`
+}
+
+type Collection struct {
+	Features []CollectionFeature `json:"features"`
+}
+
+func FetchCollection(registry, namespace string) (*Collection, error) {
+	ref, err := buildCollectionRef(registry, namespace)
+	if err != nil {
+		return nil, fmt.Errorf("parse collection reference: %w", err)
+	}
+
+	log.Debugf("fetching collection.json: registry=%s, namespace=%s", registry, namespace)
+
+	img, err := pullCollectionImage(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	return extractCollectionJSON(img)
+}
+
+func ListCollectionFeatures(registry, namespace string) ([]CollectionFeature, error) {
+	collection, err := FetchCollection(registry, namespace)
+	if err != nil {
+		return nil, err
+	}
+	return collection.Features, nil
+}
+
+func buildCollectionRef(registry, namespace string) (name.Reference, error) {
+	refStr := fmt.Sprintf("%s/%s/devcontainer-collection:latest", registry, namespace)
+	return name.ParseReference(refStr)
+}
+
+func pullCollectionImage(ref name.Reference) (v1.Image, error) {
+	var img v1.Image
+	err := retryOCIPull(func() error {
+		log.Debugf("fetching collection OCI image: reference=%s", ref.String())
+		var fetchErr error
+		img, fetchErr = remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
+		return fetchErr
+	})
+	if err != nil {
+		err = image.SanitizeRegistryError(err)
+		registry := sanitizeURL(ref.Context().RegistryStr())
+		log.Debugf("failed to fetch collection image: error=%v, registry=%s", err, registry)
+		return nil, fmt.Errorf("pull collection from %s: %w", registry, err)
+	}
+	return img, nil
+}
+
+func extractCollectionJSON(img v1.Image) (*Collection, error) {
+	layer, err := findCollectionLayer(img)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := layer.Uncompressed()
+	if err != nil {
+		return nil, fmt.Errorf("read collection layer: %w", err)
+	}
+	defer func() { _ = data.Close() }()
+
+	return parseCollection(data)
+}
+
+func findCollectionLayer(img v1.Image) (v1.Layer, error) {
+	manifest, err := img.Manifest()
+	if err != nil {
+		return nil, fmt.Errorf("read manifest: %w", err)
+	}
+
+	for _, desc := range manifest.Layers {
+		if string(desc.MediaType) == CollectionLayerMediaType {
+			layer, err := img.LayerByDigest(desc.Digest)
+			if err != nil {
+				return nil, fmt.Errorf("retrieve collection layer: %w", err)
+			}
+			return layer, nil
+		}
+	}
+
+	if len(manifest.Layers) == 0 {
+		return nil, fmt.Errorf("collection image has no layers")
+	}
+
+	log.Debugf(
+		"no layer with media type %s found, falling back to first layer",
+		CollectionLayerMediaType,
+	)
+	layer, err := img.LayerByDigest(manifest.Layers[0].Digest)
+	if err != nil {
+		return nil, fmt.Errorf("retrieve first layer: %w", err)
+	}
+	return layer, nil
+}
+
+func parseCollection(r io.Reader) (*Collection, error) {
+	raw, err := io.ReadAll(r)
+	if err != nil {
+		return nil, fmt.Errorf("read collection data: %w", err)
+	}
+
+	var collection Collection
+	if err := json.Unmarshal(raw, &collection); err != nil {
+		return nil, fmt.Errorf("parse collection.json: %w", err)
+	}
+
+	log.Debugf("parsed collection: %d features found", len(collection.Features))
+	return &collection, nil
+}

pkg/devcontainer/feature/collection_test.go

@@ -0,0 +1,226 @@
+package feature
+
+import (
+	"encoding/json"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/registry"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/static"
+	"github.com/google/go-containerregistry/pkg/v1/types"
+	"github.com/stretchr/testify/suite"
+)
+
+const testFeatureNode = "node"
+
+type CollectionTestSuite struct {
+	suite.Suite
+	server  *httptest.Server
+	regHost string
+}
+
+func TestCollectionTestSuite(t *testing.T) {
+	suite.Run(t, new(CollectionTestSuite))
+}
+
+func (s *CollectionTestSuite) SetupSuite() {
+	s.server = httptest.NewServer(registry.New())
+	s.regHost = strings.TrimPrefix(s.server.URL, "http://")
+}
+
+func (s *CollectionTestSuite) TearDownSuite() {
+	s.server.Close()
+}
+
+func (s *CollectionTestSuite) TestFetchCollection_HappyPath() {
+	collection := Collection{
+		Features: []CollectionFeature{
+			{
+				ID:          "go",
+				Version:     "1.2.3",
+				Name:        "Go",
+				Description: "Installs Go and common tools",
+			},
+			{
+				ID:          testFeatureNode,
+				Version:     "2.0.0",
+				Name:        "Node.js",
+				Description: "Installs Node.js and npm",
+				Options: map[string]any{
+					"version": map[string]any{
+						"type":    "string",
+						"default": "lts",
+					},
+				},
+			},
+		},
+	}
+
+	s.pushCollectionImage("test/ns", &collection)
+
+	result, err := FetchCollection(s.regHost, "test/ns")
+	s.Require().NoError(err)
+	s.Require().NotNil(result)
+	s.Len(result.Features, 2)
+	s.Equal("go", result.Features[0].ID)
+	s.Equal("1.2.3", result.Features[0].Version)
+	s.Equal("Go", result.Features[0].Name)
+	s.Equal(testFeatureNode, result.Features[1].ID)
+	s.Equal("2.0.0", result.Features[1].Version)
+	s.NotNil(result.Features[1].Options)
+}
+
+func (s *CollectionTestSuite) TestFetchCollection_EmptyFeatures() {
+	collection := Collection{Features: []CollectionFeature{}}
+	s.pushCollectionImage("test/empty", &collection)
+
+	result, err := FetchCollection(s.regHost, "test/empty")
+	s.Require().NoError(err)
+	s.Require().NotNil(result)
+	s.Empty(result.Features)
+}
+
+func (s *CollectionTestSuite) TestFetchCollection_RegistryNotFound() {
+	_, err := FetchCollection("localhost:1", "nonexistent/ns")
+	s.Error(err)
+	s.Contains(err.Error(), "pull collection")
+}
+
+func (s *CollectionTestSuite) TestFetchCollection_InvalidJSON() {
+	s.pushRawCollectionImage("test/badjson", []byte("not valid json"))
+
+	_, err := FetchCollection(s.regHost, "test/badjson")
+	s.Error(err)
+	s.Contains(err.Error(), "parse collection.json")
+}
+
+func (s *CollectionTestSuite) TestListCollectionFeatures() {
+	collection := Collection{
+		Features: []CollectionFeature{
+			{ID: "rust", Version: "1.0.0", Name: "Rust"},
+			{ID: "python", Version: "3.0.0", Name: "Python"},
+			{ID: "java", Version: "1.5.0", Name: "Java"},
+		},
+	}
+	s.pushCollectionImage("test/list", &collection)
+
+	features, err := ListCollectionFeatures(s.regHost, "test/list")
+	s.Require().NoError(err)
+	s.Len(features, 3)
+	s.Equal("rust", features[0].ID)
+	s.Equal("python", features[1].ID)
+	s.Equal("java", features[2].ID)
+}
+
+func (s *CollectionTestSuite) TestFetchCollection_DeprecatedFeature() {
+	collection := Collection{
+		Features: []CollectionFeature{
+			{
+				ID:         "old-feature",
+				Version:    "0.1.0",
+				Name:       "Old Feature",
+				Deprecated: true,
+			},
+		},
+	}
+	s.pushCollectionImage("test/deprecated", &collection)
+
+	result, err := FetchCollection(s.regHost, "test/deprecated")
+	s.Require().NoError(err)
+	s.True(result.Features[0].Deprecated)
+}
+
+func (s *CollectionTestSuite) TestFetchCollection_FallbackToFirstLayer() {
+	collection := Collection{
+		Features: []CollectionFeature{
+			{ID: "fallback", Version: "1.0.0", Name: "Fallback"},
+		},
+	}
+	data, err := json.Marshal(collection)
+	s.Require().NoError(err)
+
+	layer := static.NewLayer(data, types.OCILayer)
+	img, err := mutate.AppendLayers(empty.Image, layer)
+	s.Require().NoError(err)
+
+	refStr := s.regHost + "/test/fallback/devcontainer-collection:latest"
+	ref, err := name.ParseReference(refStr, name.Insecure)
+	s.Require().NoError(err)
+	s.Require().NoError(remote.Write(ref, img))
+
+	result, err := FetchCollection(s.regHost, "test/fallback")
+	s.Require().NoError(err)
+	s.Len(result.Features, 1)
+	s.Equal("fallback", result.Features[0].ID)
+}
+
+func (s *CollectionTestSuite) TestBuildCollectionRef() {
+	ref, err := buildCollectionRef("ghcr.io", "devcontainers/features")
+	s.Require().NoError(err)
+	s.Equal("ghcr.io/devcontainers/features/devcontainer-collection:latest", ref.String())
+}
+
+func (s *CollectionTestSuite) TestParseCollection_ValidJSON() {
+	input := `{"features":[{"id":"go","version":"1.0.0","name":"Go","description":"Go tools"}]}`
+	r := strings.NewReader(input)
+
+	collection, err := parseCollection(r)
+	s.Require().NoError(err)
+	s.Len(collection.Features, 1)
+	s.Equal("go", collection.Features[0].ID)
+	s.Equal("Go tools", collection.Features[0].Description)
+}
+
+func (s *CollectionTestSuite) TestParseCollection_EmptyObject() {
+	r := strings.NewReader(`{}`)
+
+	collection, err := parseCollection(r)
+	s.Require().NoError(err)
+	s.Nil(collection.Features)
+}
+
+func (s *CollectionTestSuite) pushCollectionImage(namespace string, collection *Collection) {
+	s.T().Helper()
+
+	data, err := json.Marshal(collection)
+	s.Require().NoError(err)
+
+	s.pushRawCollectionImage(namespace, data)
+}
+
+func (s *CollectionTestSuite) pushRawCollectionImage(namespace string, data []byte) {
+	s.T().Helper()
+
+	layer := static.NewLayer(data, types.MediaType(CollectionLayerMediaType))
+	img, err := mutate.AppendLayers(empty.Image, layer)
+	s.Require().NoError(err)
+
+	img = setConfigMediaType(s.T(), img)
+
+	refStr := s.regHost + "/" + namespace + "/devcontainer-collection:latest"
+	ref, err := name.ParseReference(refStr, name.Insecure)
+	s.Require().NoError(err)
+	s.Require().NoError(remote.Write(ref, img))
+}
+
+func setConfigMediaType(t *testing.T, img v1.Image) v1.Image {
+	t.Helper()
+
+	cfg, err := img.ConfigFile()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	img, err = mutate.ConfigFile(img, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return img
+}

pkg/devcontainer/feature/features_oci_test.go

@@ -34,3 +34,22 @@ func (s *OCIFeatureTestSuite) TestProcessOCIFeature_HappyPath() {
 	s.DirExists(result)
 	s.FileExists(filepath.Join(result, "devcontainer-feature.json"))
 }
+
+func (s *OCIFeatureTestSuite) TestFetchCollection_GHCR() {
+	collection, err := FetchCollection("ghcr.io", "devcontainers/features")
+	if err != nil {
+		s.T().Skipf("skipping: collection not available from ghcr.io: %v", err)
+	}
+	s.Require().NotNil(collection)
+	s.NotEmpty(collection.Features)
+
+	var foundGo bool
+	for _, f := range collection.Features {
+		s.NotEmpty(f.ID)
+		s.NotEmpty(f.Version)
+		if f.ID == "go" {
+			foundGo = true
+		}
+	}
+	s.True(foundGo, "expected 'go' feature in ghcr.io/devcontainers/features collection")
+}