diff --git a/pkg/devcontainer/feature/collection.go b/pkg/devcontainer/feature/collection.go new file mode 100644 index 000000000..4e06190de --- /dev/null +++ b/pkg/devcontainer/feature/collection.go @@ -0,0 +1,137 @@ +package feature + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/devsy-org/devsy/pkg/image" + "github.com/devsy-org/devsy/pkg/log" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +const CollectionLayerMediaType = "application/vnd.devcontainers.collection.layer.v1+json" + +type CollectionFeature struct { + ID string `json:"id"` + Version string `json:"version"` + Name string `json:"name"` + Description string `json:"description"` + DocumentationURL string `json:"documentationURL,omitempty"` + Options map[string]any `json:"options,omitempty"` + Deprecated bool `json:"deprecated,omitempty"` +} + +type Collection struct { + Features []CollectionFeature `json:"features"` +} + +func FetchCollection(registry, namespace string) (*Collection, error) { + ref, err := buildCollectionRef(registry, namespace) + if err != nil { + return nil, fmt.Errorf("parse collection reference: %w", err) + } + + log.Debugf("fetching collection.json: registry=%s, namespace=%s", registry, namespace) + + img, err := pullCollectionImage(ref) + if err != nil { + return nil, err + } + + return extractCollectionJSON(img) +} + +func ListCollectionFeatures(registry, namespace string) ([]CollectionFeature, error) { + collection, err := FetchCollection(registry, namespace) + if err != nil { + return nil, err + } + return collection.Features, nil +} + +func buildCollectionRef(registry, namespace string) (name.Reference, error) { + refStr := fmt.Sprintf("%s/%s/devcontainer-collection:latest", registry, namespace) + return name.ParseReference(refStr) +} + +func pullCollectionImage(ref name.Reference) (v1.Image, error) { + var img v1.Image + err := retryOCIPull(func() error { + log.Debugf("fetching collection OCI image: reference=%s", ref.String()) + var fetchErr error + img, fetchErr = remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + return fetchErr + }) + if err != nil { + err = image.SanitizeRegistryError(err) + registry := sanitizeURL(ref.Context().RegistryStr()) + log.Debugf("failed to fetch collection image: error=%v, registry=%s", err, registry) + return nil, fmt.Errorf("pull collection from %s: %w", registry, err) + } + return img, nil +} + +func extractCollectionJSON(img v1.Image) (*Collection, error) { + layer, err := findCollectionLayer(img) + if err != nil { + return nil, err + } + + data, err := layer.Uncompressed() + if err != nil { + return nil, fmt.Errorf("read collection layer: %w", err) + } + defer func() { _ = data.Close() }() + + return parseCollection(data) +} + +func findCollectionLayer(img v1.Image) (v1.Layer, error) { + manifest, err := img.Manifest() + if err != nil { + return nil, fmt.Errorf("read manifest: %w", err) + } + + for _, desc := range manifest.Layers { + if string(desc.MediaType) == CollectionLayerMediaType { + layer, err := img.LayerByDigest(desc.Digest) + if err != nil { + return nil, fmt.Errorf("retrieve collection layer: %w", err) + } + return layer, nil + } + } + + if len(manifest.Layers) == 0 { + return nil, fmt.Errorf("collection image has no layers") + } + + log.Debugf( + "no layer with media type %s found, falling back to first layer", + CollectionLayerMediaType, + ) + layer, err := img.LayerByDigest(manifest.Layers[0].Digest) + if err != nil { + return nil, fmt.Errorf("retrieve first layer: %w", err) + } + return layer, nil +} + +func parseCollection(r io.Reader) (*Collection, error) { + raw, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("read collection data: %w", err) + } + + var collection Collection + if err := json.Unmarshal(raw, &collection); err != nil { + return nil, fmt.Errorf("parse collection.json: %w", err) + } + + log.Debugf("parsed collection: %d features found", len(collection.Features)) + return &collection, nil +} diff --git a/pkg/devcontainer/feature/collection_test.go b/pkg/devcontainer/feature/collection_test.go new file mode 100644 index 000000000..0c7afa2a8 --- /dev/null +++ b/pkg/devcontainer/feature/collection_test.go @@ -0,0 +1,226 @@ +package feature + +import ( + "encoding/json" + "net/http/httptest" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/stretchr/testify/suite" +) + +const testFeatureNode = "node" + +type CollectionTestSuite struct { + suite.Suite + server *httptest.Server + regHost string +} + +func TestCollectionTestSuite(t *testing.T) { + suite.Run(t, new(CollectionTestSuite)) +} + +func (s *CollectionTestSuite) SetupSuite() { + s.server = httptest.NewServer(registry.New()) + s.regHost = strings.TrimPrefix(s.server.URL, "http://") +} + +func (s *CollectionTestSuite) TearDownSuite() { + s.server.Close() +} + +func (s *CollectionTestSuite) TestFetchCollection_HappyPath() { + collection := Collection{ + Features: []CollectionFeature{ + { + ID: "go", + Version: "1.2.3", + Name: "Go", + Description: "Installs Go and common tools", + }, + { + ID: testFeatureNode, + Version: "2.0.0", + Name: "Node.js", + Description: "Installs Node.js and npm", + Options: map[string]any{ + "version": map[string]any{ + "type": "string", + "default": "lts", + }, + }, + }, + }, + } + + s.pushCollectionImage("test/ns", &collection) + + result, err := FetchCollection(s.regHost, "test/ns") + s.Require().NoError(err) + s.Require().NotNil(result) + s.Len(result.Features, 2) + s.Equal("go", result.Features[0].ID) + s.Equal("1.2.3", result.Features[0].Version) + s.Equal("Go", result.Features[0].Name) + s.Equal(testFeatureNode, result.Features[1].ID) + s.Equal("2.0.0", result.Features[1].Version) + s.NotNil(result.Features[1].Options) +} + +func (s *CollectionTestSuite) TestFetchCollection_EmptyFeatures() { + collection := Collection{Features: []CollectionFeature{}} + s.pushCollectionImage("test/empty", &collection) + + result, err := FetchCollection(s.regHost, "test/empty") + s.Require().NoError(err) + s.Require().NotNil(result) + s.Empty(result.Features) +} + +func (s *CollectionTestSuite) TestFetchCollection_RegistryNotFound() { + _, err := FetchCollection("localhost:1", "nonexistent/ns") + s.Error(err) + s.Contains(err.Error(), "pull collection") +} + +func (s *CollectionTestSuite) TestFetchCollection_InvalidJSON() { + s.pushRawCollectionImage("test/badjson", []byte("not valid json")) + + _, err := FetchCollection(s.regHost, "test/badjson") + s.Error(err) + s.Contains(err.Error(), "parse collection.json") +} + +func (s *CollectionTestSuite) TestListCollectionFeatures() { + collection := Collection{ + Features: []CollectionFeature{ + {ID: "rust", Version: "1.0.0", Name: "Rust"}, + {ID: "python", Version: "3.0.0", Name: "Python"}, + {ID: "java", Version: "1.5.0", Name: "Java"}, + }, + } + s.pushCollectionImage("test/list", &collection) + + features, err := ListCollectionFeatures(s.regHost, "test/list") + s.Require().NoError(err) + s.Len(features, 3) + s.Equal("rust", features[0].ID) + s.Equal("python", features[1].ID) + s.Equal("java", features[2].ID) +} + +func (s *CollectionTestSuite) TestFetchCollection_DeprecatedFeature() { + collection := Collection{ + Features: []CollectionFeature{ + { + ID: "old-feature", + Version: "0.1.0", + Name: "Old Feature", + Deprecated: true, + }, + }, + } + s.pushCollectionImage("test/deprecated", &collection) + + result, err := FetchCollection(s.regHost, "test/deprecated") + s.Require().NoError(err) + s.True(result.Features[0].Deprecated) +} + +func (s *CollectionTestSuite) TestFetchCollection_FallbackToFirstLayer() { + collection := Collection{ + Features: []CollectionFeature{ + {ID: "fallback", Version: "1.0.0", Name: "Fallback"}, + }, + } + data, err := json.Marshal(collection) + s.Require().NoError(err) + + layer := static.NewLayer(data, types.OCILayer) + img, err := mutate.AppendLayers(empty.Image, layer) + s.Require().NoError(err) + + refStr := s.regHost + "/test/fallback/devcontainer-collection:latest" + ref, err := name.ParseReference(refStr, name.Insecure) + s.Require().NoError(err) + s.Require().NoError(remote.Write(ref, img)) + + result, err := FetchCollection(s.regHost, "test/fallback") + s.Require().NoError(err) + s.Len(result.Features, 1) + s.Equal("fallback", result.Features[0].ID) +} + +func (s *CollectionTestSuite) TestBuildCollectionRef() { + ref, err := buildCollectionRef("ghcr.io", "devcontainers/features") + s.Require().NoError(err) + s.Equal("ghcr.io/devcontainers/features/devcontainer-collection:latest", ref.String()) +} + +func (s *CollectionTestSuite) TestParseCollection_ValidJSON() { + input := `{"features":[{"id":"go","version":"1.0.0","name":"Go","description":"Go tools"}]}` + r := strings.NewReader(input) + + collection, err := parseCollection(r) + s.Require().NoError(err) + s.Len(collection.Features, 1) + s.Equal("go", collection.Features[0].ID) + s.Equal("Go tools", collection.Features[0].Description) +} + +func (s *CollectionTestSuite) TestParseCollection_EmptyObject() { + r := strings.NewReader(`{}`) + + collection, err := parseCollection(r) + s.Require().NoError(err) + s.Nil(collection.Features) +} + +func (s *CollectionTestSuite) pushCollectionImage(namespace string, collection *Collection) { + s.T().Helper() + + data, err := json.Marshal(collection) + s.Require().NoError(err) + + s.pushRawCollectionImage(namespace, data) +} + +func (s *CollectionTestSuite) pushRawCollectionImage(namespace string, data []byte) { + s.T().Helper() + + layer := static.NewLayer(data, types.MediaType(CollectionLayerMediaType)) + img, err := mutate.AppendLayers(empty.Image, layer) + s.Require().NoError(err) + + img = setConfigMediaType(s.T(), img) + + refStr := s.regHost + "/" + namespace + "/devcontainer-collection:latest" + ref, err := name.ParseReference(refStr, name.Insecure) + s.Require().NoError(err) + s.Require().NoError(remote.Write(ref, img)) +} + +func setConfigMediaType(t *testing.T, img v1.Image) v1.Image { + t.Helper() + + cfg, err := img.ConfigFile() + if err != nil { + t.Fatal(err) + } + + img, err = mutate.ConfigFile(img, cfg) + if err != nil { + t.Fatal(err) + } + + return img +} diff --git a/pkg/devcontainer/feature/features_oci_test.go b/pkg/devcontainer/feature/features_oci_test.go index 8d297ca78..4593da0b7 100644 --- a/pkg/devcontainer/feature/features_oci_test.go +++ b/pkg/devcontainer/feature/features_oci_test.go @@ -34,3 +34,22 @@ func (s *OCIFeatureTestSuite) TestProcessOCIFeature_HappyPath() { s.DirExists(result) s.FileExists(filepath.Join(result, "devcontainer-feature.json")) } + +func (s *OCIFeatureTestSuite) TestFetchCollection_GHCR() { + collection, err := FetchCollection("ghcr.io", "devcontainers/features") + if err != nil { + s.T().Skipf("skipping: collection not available from ghcr.io: %v", err) + } + s.Require().NotNil(collection) + s.NotEmpty(collection.Features) + + var foundGo bool + for _, f := range collection.Features { + s.NotEmpty(f.ID) + s.NotEmpty(f.Version) + if f.ID == "go" { + foundGo = true + } + } + s.True(foundGo, "expected 'go' feature in ghcr.io/devcontainers/features collection") +}