Update ocimem to be compatible with non-canonical algorithms

ocimem/blob.go

@@ -21,7 +21,6 @@ import (
 	"sync"
 
 	"github.com/docker/oci"
-	"github.com/docker/oci/ocidigest"
 )
 
 // NewBytesReader returns an implementation of oci.BlobReader
@@ -65,9 +64,9 @@ type Buffer struct {
 	commitErr        error
 }
 
-// NewBuffer returns a buffer that calls commit with the
-// when [Buffer.Commit] is invoked successfully.
-// /
+// NewBuffer returns a buffer that calls commit when [Buffer.Commit]
+// is invoked successfully.
+//
 // It's OK to call methods concurrently on a buffer.
 func NewBuffer(commit func(b *Buffer) error, uuid string) *Buffer {
 	if uuid == "" {
@@ -104,7 +103,7 @@ func (b *Buffer) ChunkSize() int {
 	return 8 * 1024 // 8KiB; not really important
 }
 
-// GetBlob returns any committed data and is descriptor. It returns an error
+// GetBlob returns any committed data and its descriptor. It returns an error
 // if the data hasn't been committed or there was an error doing so.
 func (b *Buffer) GetBlob() (oci.Descriptor, []byte, error) {
 	b.mu.Lock()
@@ -181,8 +180,8 @@ func (b *Buffer) checkCommit(dig oci.Digest) (err error) {
 			b.commitErr = err
 		}
 	}()
-	if ocidigest.FromBytes(b.buf) != dig {
-		return fmt.Errorf("digest mismatch (sha256(%q) != %s): %w", b.buf, dig, oci.ErrDigestInvalid)
+	if got := dig.Algorithm().FromBytes(b.buf); got != dig {
+		return fmt.Errorf("digest mismatch (%s(%q) = %s, want %s): %w", dig.Algorithm(), b.buf, got, dig, oci.ErrDigestInvalid)
 	}
 	b.desc = oci.Descriptor{
 		MediaType: "application/octet-stream",

ocimem/check_test.go

@@ -1,12 +1,15 @@
 package ocimem
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
+	"io"
 	"testing"
 
 	"github.com/docker/oci"
+	"github.com/docker/oci/ocidigest"
 	"github.com/docker/oci/ocitest"
 	"github.com/stretchr/testify/require"
 )
@@ -53,6 +56,26 @@ var pushManifestTests = []struct {
 		})
 	},
 	wantError: `invalid manifest: blob for layers\[0\] not found`,
+}, {
+	testName: "NonExistentLayerReferenceWithURLs",
+	preload: ocitest.RepoContent{
+		Blobs: map[string]string{
+			"a": "{}",
+		},
+	},
+	mediaType: oci.MediaTypeImageManifest,
+	manifestData: func(content ocitest.PushedRepoContent) []byte {
+		return mustJSONMarshal(oci.IndexOrManifest{
+			MediaType: oci.MediaTypeImageManifest,
+			Config:    ref(content.Blobs["a"]),
+			Layers: []oci.Descriptor{{
+				MediaType: "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+				Size:      1,
+				Digest:    ocitest.DigestRef("b"),
+				URLs:      []string{"https://example.com/foreign-layer"},
+			}},
+		})
+	},
 }, {
 	testName: "NonExistentSubjectReference",
 	preload: ocitest.RepoContent{
@@ -284,6 +307,121 @@ func TestPushManifest(t *testing.T) {
 	}
 }
 
+func TestNonCanonicalPushBlob(t *testing.T) {
+	ctx := context.Background()
+	r := New()
+	data := []byte("blob data")
+	digest := ocidigest.SHA512.FromBytes(data)
+	desc := oci.Descriptor{
+		MediaType: "application/octet-stream",
+		Digest:    digest,
+		Size:      int64(len(data)),
+	}
+
+	gotDesc, err := r.PushBlob(ctx, "test", desc, bytes.NewReader(data))
+	require.NoError(t, err)
+	require.Equal(t, desc, gotDesc)
+
+	resolved, err := r.ResolveBlob(ctx, "test", digest)
+	require.NoError(t, err)
+	require.Equal(t, desc, resolved)
+
+	br, err := r.GetBlob(ctx, "test", digest)
+	require.NoError(t, err)
+	defer br.Close()
+	require.Equal(t, desc, br.Descriptor())
+	gotData, err := io.ReadAll(br)
+	require.NoError(t, err)
+	require.Equal(t, data, gotData)
+}
+
+func TestNonCanonicalPushBlobRejectsDigestMismatch(t *testing.T) {
+	ctx := context.Background()
+	r := New()
+	data := []byte("blob data")
+	desc := oci.Descriptor{
+		MediaType: "application/octet-stream",
+		Digest:    ocidigest.SHA512.FromBytes([]byte("other data")),
+		Size:      int64(len(data)),
+	}
+
+	_, err := r.PushBlob(ctx, "test", desc, bytes.NewReader(data))
+	require.Error(t, err)
+	require.Regexp(t, `digest invalid: provided digest did not match uploaded content`, err.Error())
+}
+
+func TestNonCanonicalPushBlobChunked(t *testing.T) {
+	ctx := context.Background()
+	r := New()
+	data := []byte("chunked blob data")
+	digest := ocidigest.SHA512.FromBytes(data)
+
+	w, err := r.PushBlobChunked(ctx, "test", 0)
+	require.NoError(t, err)
+	_, err = w.Write(data[:7])
+	require.NoError(t, err)
+	_, err = w.Write(data[7:])
+	require.NoError(t, err)
+	desc, err := w.Commit(digest)
+	require.NoError(t, err)
+	require.Equal(t, digest, desc.Digest)
+	require.Equal(t, int64(len(data)), desc.Size)
+
+	resolved, err := r.ResolveBlob(ctx, "test", digest)
+	require.NoError(t, err)
+	require.Equal(t, desc, resolved)
+}
+
+func TestNonCanonicalPushManifest(t *testing.T) {
+	ctx := context.Background()
+	r := New()
+	configData := []byte("{}")
+	configDigest := ocidigest.SHA512.FromBytes(configData)
+	configDesc := oci.Descriptor{
+		MediaType: "application/vnd.example.config",
+		Digest:    configDigest,
+		Size:      int64(len(configData)),
+	}
+	_, err := r.PushBlob(ctx, "test", configDesc, bytes.NewReader(configData))
+	require.NoError(t, err)
+
+	manifest := oci.IndexOrManifest{
+		MediaType: oci.MediaTypeImageManifest,
+		Config:    ref(configDesc),
+	}
+	data := mustJSONMarshal(manifest)
+	manifestDigest := ocidigest.SHA512.FromBytes(data)
+	manifestDesc := oci.Descriptor{
+		MediaType: oci.MediaTypeImageManifest,
+		Digest:    manifestDigest,
+		Size:      int64(len(data)),
+	}
+	desc, err := r.PushManifest(ctx, "test", data, oci.MediaTypeImageManifest, &oci.PushManifestParameters{
+		Digest: manifestDigest,
+		Tags:   []string{"sha512"},
+	})
+	require.NoError(t, err)
+	require.Equal(t, manifestDesc, desc)
+	storedManifestDesc := manifestDesc
+	storedManifestDesc.ArtifactType = configDesc.MediaType
+
+	resolved, err := r.ResolveManifest(ctx, "test", manifestDigest)
+	require.NoError(t, err)
+	require.Equal(t, storedManifestDesc, resolved)
+
+	tagDesc, err := r.ResolveTag(ctx, "test", "sha512")
+	require.NoError(t, err)
+	require.Equal(t, manifestDesc, tagDesc)
+
+	mr, err := r.GetTag(ctx, "test", "sha512")
+	require.NoError(t, err)
+	defer mr.Close()
+	require.Equal(t, storedManifestDesc, mr.Descriptor())
+	gotData, err := io.ReadAll(mr)
+	require.NoError(t, err)
+	require.Equal(t, data, gotData)
+}
+
 var deleteBlobTests = []struct {
 	testName  string
 	config    Config

ocimem/registry.go

@@ -21,7 +21,6 @@ import (
 	"sync"
 
 	"github.com/docker/oci"
-	"github.com/docker/oci/ocidigest"
 	"github.com/docker/oci/ociref"
 )
 
@@ -43,6 +42,7 @@ type repository struct {
 }
 
 type blob struct {
+	digest    oci.Digest
 	mediaType string
 	data      []byte
 	info      manifestInfo
@@ -52,7 +52,7 @@ func (b *blob) descriptor() oci.Descriptor {
 	return oci.Descriptor{
 		MediaType:    b.mediaType,
 		Size:         int64(len(b.data)),
-		Digest:       ocidigest.FromBytes(b.data),
+		Digest:       b.digest,
 		ArtifactType: b.info.artifactType,
 		Annotations:  b.info.annotations,
 	}
@@ -150,25 +150,22 @@ func (r *Registry) makeRepo(repoName string) (*repository, error) {
 	return repo, nil
 }
 
-// SHA256("")
-var emptyHash = ocidigest.FromBytes(nil)
-
 // CheckDescriptor checks that the given descriptor matches the given data or,
 // if data is nil, that the descriptor looks sane.
 func CheckDescriptor(desc oci.Descriptor, data []byte) error {
 	if err := desc.Digest.Validate(); err != nil {
 		return fmt.Errorf("invalid digest: %v", err)
 	}
 	if data != nil {
-		if ocidigest.FromBytes(data) != desc.Digest {
-			return fmt.Errorf("digest mismatch")
+		if desc.Digest.Algorithm().FromBytes(data) != desc.Digest {
+			return fmt.Errorf("digest mismatch: %w", oci.ErrDigestInvalid)
 		}
 		if desc.Size != int64(len(data)) {
-			return fmt.Errorf("size mismatch")
+			return fmt.Errorf("size mismatch: %w", oci.ErrSizeInvalid)
 		}
 	} else {
-		if desc.Size == 0 && desc.Digest != emptyHash {
-			return fmt.Errorf("zero sized content with mismatching digest")
+		if desc.Size == 0 && desc.Digest.Algorithm().FromBytes(nil) != desc.Digest {
+			return fmt.Errorf("zero sized content with mismatching digest: %w", oci.ErrDigestInvalid)
 		}
 	}
 	if desc.MediaType == "" {

ocimem/writer.go

@@ -43,7 +43,7 @@ func (r *Registry) PushBlob(ctx context.Context, repoName string, desc oci.Descr
 	if err != nil {
 		return oci.Descriptor{}, err
 	}
-	repo.blobs[desc.Digest] = &blob{mediaType: desc.MediaType, data: data}
+	repo.blobs[desc.Digest] = &blob{digest: desc.Digest, mediaType: desc.MediaType, data: data}
 	return desc, nil
 }
 
@@ -72,7 +72,7 @@ func (r *Registry) PushBlobChunkedResume(ctx context.Context, repoName, id strin
 			r.mu.Lock()
 			defer r.mu.Unlock()
 			desc, data, _ := b.GetBlob()
-			repo.blobs[desc.Digest] = &blob{mediaType: desc.MediaType, data: data}
+			repo.blobs[desc.Digest] = &blob{digest: desc.Digest, mediaType: desc.MediaType, data: data}
 			return nil
 		}, id)
 		repo.uploads[b.ID()] = b
@@ -179,6 +179,7 @@ func (r *Registry) PushManifest(ctx context.Context, repoName string, data []byt
 	}
 
 	repo.manifests[dig] = &blob{
+		digest:    dig,
 		mediaType: mediaType,
 		data:      data,
 		info:      info,
@@ -208,6 +209,9 @@ func (r *Registry) checkManifestReferences(repoName string, mediaType string, da
 		}
 		switch info.kind {
 		case kindBlob:
+			if len(info.desc.URLs) > 0 {
+				continue
+			}
 			if repo.blobs[info.desc.Digest] == nil {
 				return manifestInfo{}, fmt.Errorf("blob for %s not found", info.name)
 			}