Ocilayout impl

README.md

@@ -28,6 +28,7 @@ used to interact with those features.
 | `ociclient` | HTTP client that implements `oci.Interface` against a remote OCI registry. |
 | `ociserver` | HTTP server that serves the OCI distribution protocol on top of any `oci.Interface`. |
 | `ocimem` | Lightweight in-memory `oci.Interface` implementation, useful for testing and caching. |
+| `ocilayout` | Filesystem-backed `oci.Interface` implementation for OCI Image Layout directories, including shared and per-repository layouts. |
 | `ociauth` | Authentication transport implementing the Docker/OCI token flow, plus helpers for loading credentials from Docker config files. |
 | `ocifilter` | Wrappers that expose restricted or transformed views of a registry (read-only, immutable, namespace prefix, custom access control). |
 | `ociunify` | Combines two registries into a single unified `oci.Interface`, with configurable read policy. |

ocilayout/README.md

@@ -0,0 +1,94 @@
+# ocilayout
+
+Package `ocilayout` provides an `oci.Interface` implementation backed by an
+[OCI Image Layout](https://github.com/opencontainers/image-spec/blob/main/image-layout.md)
+directory on disk.
+
+Use it when you want registry-like reads and writes without running a registry
+server. It stores manifests in `index.json`, blobs under `blobs/`, and records
+named references with the standard `org.opencontainers.image.ref.name`
+annotation.
+
+## Shared Layout
+
+`New` opens a single OCI layout directory that can hold multiple repositories.
+Missing layout files are not created when the registry is opened; write
+operations create `oci-layout`, `index.json`, and blob directories lazily.
+
+```go
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/docker/oci"
+	"github.com/docker/oci/ocilayout"
+)
+
+func main() {
+	reg, err := ocilayout.New("./layout", &ocilayout.Options{})
+	if err != nil {
+		panic(err)
+	}
+
+	tags, err := oci.All(reg.Tags(context.Background(), "example/app", nil))
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println(tags)
+}
+```
+
+When reading existing layouts that use tag-only `ref.name` annotations, set
+`DefaultRepo` so those entries can be associated with a repository.
+
+```go
+reg, err := ocilayout.New("./layout", &ocilayout.Options{
+	DefaultRepo: "example/app",
+})
+```
+
+## Per-Repository Layouts
+
+`NewPerRepository` stores each repository in its own nested OCI layout under the
+given directory. For repository `example/app`, the underlying layout lives at
+`<dir>/example/app`.
+
+```go
+reg, err := ocilayout.NewPerRepository("./layouts", &ocilayout.PerRepoOptions{})
+if err != nil {
+	panic(err)
+}
+```
+
+The per-repository constructor has its own options type so its API can evolve
+independently from `New`.
+
+## Finding Layouts From Paths
+
+`FindLayout` splits a user-supplied path into the base directory for `New` and
+the image reference suffix. It first looks for `oci-layout` marker files in path
+prefixes and uses the deepest matching layout. If no marker exists, it falls
+back to treating the last path component as the reference.
+
+```go
+baseDir, ref, err := ocilayout.FindLayout("./foo/bar:baz")
+// baseDir == "./foo"
+// ref.Repository == "bar"
+// ref.Tag == "baz"
+```
+
+```go
+baseDir, ref, err := ocilayout.FindLayout("./one/two/three/four:tag")
+// If ./one/two/oci-layout exists:
+// baseDir == "./one/two"
+// ref.Repository == "three/four"
+// ref.Tag == "tag"
+```
+
+References may include tags, digests, or both:
+
+```go
+baseDir, ref, err := ocilayout.FindLayout("./layout/repo:tag@sha256:...")
+```

ocilayout/blob.go

@@ -0,0 +1,179 @@
+// Copyright 2023 CUE Labs AG
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ocilayout
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/oci"
+	"github.com/docker/oci/ocidigest"
+)
+
+func blobPath(dir string, digest oci.Digest) (string, error) {
+	if err := digest.Validate(); err != nil {
+		return "", fmt.Errorf("%w: %v", oci.ErrDigestInvalid, err)
+	}
+	alg := digest.Algorithm().String()
+	enc := digest.Encoded()
+	if alg == "" || enc == "" {
+		return "", oci.ErrDigestInvalid
+	}
+	return filepath.Join(dir, "blobs", alg, enc), nil
+}
+
+func ensureBlobExists(dir string, digest oci.Digest) error {
+	path, err := blobPath(dir, digest)
+	if err != nil {
+		return err
+	}
+	_, err = os.Stat(path) // #nosec G703 -- path is derived from a validated OCI digest.
+	return err
+}
+
+func writeBlob(dir string, desc oci.Descriptor, r io.Reader) (oci.Descriptor, error) {
+	if desc.Size < 0 {
+		return oci.Descriptor{}, oci.ErrSizeInvalid
+	}
+	final, err := blobPath(dir, desc.Digest)
+	if err != nil {
+		return oci.Descriptor{}, err
+	}
+	if existing, err := os.Stat(final); err == nil { // #nosec G703 -- path is derived from a validated OCI digest.
+		if existing.Size() != desc.Size {
+			return oci.Descriptor{}, oci.ErrSizeInvalid
+		}
+		return desc, nil
+	} else if !os.IsNotExist(err) {
+		return oci.Descriptor{}, err
+	}
+	if err := os.MkdirAll(filepath.Join(dir, "blobs", "uploads"), 0o700); err != nil {
+		return oci.Descriptor{}, err
+	}
+	tmp := filepath.Join(dir, "blobs", "uploads", newUploadID())
+	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
+	if err != nil {
+		return oci.Descriptor{}, err
+	}
+	cleanup := true
+	defer func() {
+		f.Close()
+		if cleanup {
+			os.Remove(tmp)
+		}
+	}()
+	dw, err := ocidigest.NewWriter(f, desc.Digest.Algorithm())
+	if err != nil {
+		return oci.Descriptor{}, err
+	}
+	if _, err := io.Copy(dw, r); err != nil {
+		return oci.Descriptor{}, err
+	}
+	got, err := dw.Digest()
+	if err != nil {
+		return oci.Descriptor{}, err
+	}
+	if got != desc.Digest {
+		return oci.Descriptor{}, fmt.Errorf("digest mismatch: %w", oci.ErrDigestInvalid)
+	}
+	if dw.Size() != desc.Size {
+		return oci.Descriptor{}, fmt.Errorf("size mismatch: %w", oci.ErrSizeInvalid)
+	}
+	if err := f.Close(); err != nil {
+		return oci.Descriptor{}, err
+	}
+	if err := os.MkdirAll(filepath.Dir(final), 0o700); err != nil { // #nosec G703 -- path is derived from a validated OCI digest.
+		return oci.Descriptor{}, err
+	}
+	if err := os.Rename(tmp, final); err != nil { // #nosec G703 -- final path is derived from a validated OCI digest.
+		return oci.Descriptor{}, err
+	}
+	cleanup = false
+	return desc, nil
+}
+
+func writeBlobBytes(dir string, desc oci.Descriptor, data []byte) (oci.Descriptor, error) {
+	return writeBlob(dir, desc, bytes.NewReader(data))
+}
+
+type layoutReader struct {
+	*os.File
+	desc oci.Descriptor
+}
+
+func (r *layoutReader) Descriptor() oci.Descriptor {
+	return r.desc
+}
+
+type sectionReadCloser struct {
+	*io.SectionReader
+	f    *os.File
+	desc oci.Descriptor
+}
+
+func (r *sectionReadCloser) Close() error {
+	return r.f.Close()
+}
+
+func (r *sectionReadCloser) Descriptor() oci.Descriptor {
+	return r.desc
+}
+
+func openBlob(dir string, desc oci.Descriptor) (oci.BlobReader, error) {
+	path, err := blobPath(dir, desc.Digest)
+	if err != nil {
+		return nil, err
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	return &layoutReader{File: f, desc: desc}, nil
+}
+
+func openBlobRange(dir string, desc oci.Descriptor, o0, o1 int64) (oci.BlobReader, error) {
+	if o1 < 0 || o1 > desc.Size {
+		o1 = desc.Size
+	}
+	if o0 < 0 || o0 > o1 {
+		return nil, fmt.Errorf("invalid range [%d, %d]; have [%d, %d]: %w", o0, o1, 0, desc.Size, oci.ErrRangeInvalid)
+	}
+	path, err := blobPath(dir, desc.Digest)
+	if err != nil {
+		return nil, err
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	return &sectionReadCloser{
+		SectionReader: io.NewSectionReader(f, o0, o1-o0),
+		f:             f,
+		desc:          desc,
+	}, nil
+}
+
+func newUploadID() string {
+	var buf [16]byte
+	if _, err := rand.Read(buf[:]); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(buf[:])
+}