diff --git a/cmd/entire/cli/checkpoint/blob_resolver.go b/cmd/entire/cli/checkpoint/blob_resolver.go new file mode 100644 index 0000000000..7241738db6 --- /dev/null +++ b/cmd/entire/cli/checkpoint/blob_resolver.go @@ -0,0 +1,126 @@ +package checkpoint + +import ( + "fmt" + "io" + "strconv" + "strings" + + "github.com/entireio/cli/cmd/entire/cli/agent" + "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" + "github.com/entireio/cli/cmd/entire/cli/paths" + + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/plumbing/storer" +) + +// TranscriptBlobRef identifies a blob within a checkpoint tree on the metadata branch. +// It captures the blob hash from the tree entry without requiring the blob itself to be local. +type TranscriptBlobRef struct { + // SessionIndex is the 0-based session index within the checkpoint. + SessionIndex int + + // Hash is the blob's SHA-1 hash from the tree entry. + Hash plumbing.Hash + + // Path is the blob's path relative to the checkpoint directory, + // e.g. "0/full.jsonl" or "0/full.jsonl.001". + Path string +} + +// BlobResolver checks blob existence and reads blobs from go-git's local +// object store (loose objects + packfiles). It performs no remote operations. +type BlobResolver struct { + storer storer.EncodedObjectStorer +} + +// NewBlobResolver creates a BlobResolver backed by the given object store. +func NewBlobResolver(s storer.EncodedObjectStorer) *BlobResolver { + return &BlobResolver{storer: s} +} + +// HasBlob returns true if the blob exists in the local object store. +// Checks both loose objects and packfile indices without reading blob content. +func (r *BlobResolver) HasBlob(hash plumbing.Hash) bool { + return r.storer.HasEncodedObject(hash) == nil +} + +// ReadBlob reads a blob's content from the local object store. +// Returns plumbing.ErrObjectNotFound if the blob is not present locally. +func (r *BlobResolver) ReadBlob(hash plumbing.Hash) ([]byte, error) { + obj, err := r.storer.EncodedObject(plumbing.BlobObject, hash) + if err != nil { + return nil, err //nolint:wrapcheck // Propagating plumbing.ErrObjectNotFound + } + + reader, err := obj.Reader() + if err != nil { + return nil, fmt.Errorf("blob reader %s: %w", hash, err) + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("read blob %s: %w", hash, err) + } + return data, nil +} + +// CollectTranscriptBlobHashes walks the metadata branch tree for a checkpoint +// and returns blob hashes for all transcript files (full.jsonl and chunks) +// across all sessions. Only reads tree objects — works after a treeless fetch +// where blobs have not been downloaded. +// +// The function navigates the sharded checkpoint directory structure: +// +// // +// ├── 0/ +// │ ├── full.jsonl ← collected +// │ ├── full.jsonl.001 ← collected (chunk) +// │ └── metadata.json +// ├── 1/ +// │ └── full.jsonl ← collected +// └── metadata.json +func CollectTranscriptBlobHashes(tree *object.Tree, checkpointID id.CheckpointID) ([]TranscriptBlobRef, error) { + checkpointTree, err := tree.Tree(checkpointID.Path()) + if err != nil { + return nil, fmt.Errorf("checkpoint tree %s: %w", checkpointID.Path(), err) + } + + var refs []TranscriptBlobRef + + // Enumerate session subdirectories (0, 1, 2, ...) + for i := 0; ; i++ { + sessionDir := strconv.Itoa(i) + sessionTree, treeErr := checkpointTree.Tree(sessionDir) + if treeErr != nil { + break // no more sessions + } + + // Collect transcript blob hashes from tree entries. + // tree.Entries contains the direct children — no blob reads needed. + for _, entry := range sessionTree.Entries { + if entry.Name == paths.TranscriptFileName || entry.Name == paths.TranscriptFileNameLegacy { + refs = append(refs, TranscriptBlobRef{ + SessionIndex: i, + Hash: entry.Hash, + Path: sessionDir + "/" + entry.Name, + }) + } + // Check for chunk files (full.jsonl.001, full.jsonl.002, etc.) + if strings.HasPrefix(entry.Name, paths.TranscriptFileName+".") { + idx := agent.ParseChunkIndex(entry.Name, paths.TranscriptFileName) + if idx > 0 { + refs = append(refs, TranscriptBlobRef{ + SessionIndex: i, + Hash: entry.Hash, + Path: sessionDir + "/" + entry.Name, + }) + } + } + } + } + + return refs, nil //nolint:nilerr // treeErr from session enumeration loop is used to break, not propagated +} diff --git a/cmd/entire/cli/checkpoint/blob_resolver_test.go b/cmd/entire/cli/checkpoint/blob_resolver_test.go new file mode 100644 index 0000000000..2b6787758c --- /dev/null +++ b/cmd/entire/cli/checkpoint/blob_resolver_test.go @@ -0,0 +1,200 @@ +package checkpoint + +import ( + "context" + "testing" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" + + "github.com/go-git/go-git/v6/plumbing" +) + +func TestBlobResolver_HasBlob_Present(t *testing.T) { + t.Parallel() + + repo, store, cpID := setupRepoForUpdate(t) + + // Get the metadata branch tree + tree, err := store.getSessionsBranchTree() + if err != nil { + t.Fatalf("getSessionsBranchTree() error = %v", err) + } + + // Navigate to the transcript blob via tree entries + refs, err := CollectTranscriptBlobHashes(tree, cpID) + if err != nil { + t.Fatalf("CollectTranscriptBlobHashes() error = %v", err) + } + if len(refs) == 0 { + t.Fatal("expected at least one transcript blob ref") + } + + resolver := NewBlobResolver(repo.Storer) + + // Blob should exist — it was written by WriteCommitted + if !resolver.HasBlob(refs[0].Hash) { + t.Errorf("HasBlob(%s) = false, want true (blob was written locally)", refs[0].Hash) + } +} + +func TestBlobResolver_HasBlob_Missing(t *testing.T) { + t.Parallel() + + repo, _, _ := setupRepoForUpdate(t) + resolver := NewBlobResolver(repo.Storer) + + // Random hash that doesn't exist + fakeHash := plumbing.NewHash("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef") + if resolver.HasBlob(fakeHash) { + t.Error("HasBlob(fake) = true, want false") + } +} + +func TestBlobResolver_ReadBlob(t *testing.T) { + t.Parallel() + + repo, store, cpID := setupRepoForUpdate(t) + + tree, err := store.getSessionsBranchTree() + if err != nil { + t.Fatalf("getSessionsBranchTree() error = %v", err) + } + + refs, err := CollectTranscriptBlobHashes(tree, cpID) + if err != nil { + t.Fatalf("CollectTranscriptBlobHashes() error = %v", err) + } + if len(refs) == 0 { + t.Fatal("expected at least one transcript blob ref") + } + + resolver := NewBlobResolver(repo.Storer) + + data, err := resolver.ReadBlob(refs[0].Hash) + if err != nil { + t.Fatalf("ReadBlob() error = %v", err) + } + if len(data) == 0 { + t.Error("ReadBlob() returned empty data") + } + // The transcript content from setupRepoForUpdate + if string(data) != "provisional transcript line 1\n" { + t.Errorf("ReadBlob() = %q, want %q", string(data), "provisional transcript line 1\n") + } +} + +func TestBlobResolver_ReadBlob_Missing(t *testing.T) { + t.Parallel() + + repo, _, _ := setupRepoForUpdate(t) + resolver := NewBlobResolver(repo.Storer) + + fakeHash := plumbing.NewHash("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef") + _, err := resolver.ReadBlob(fakeHash) + if err == nil { + t.Error("ReadBlob(fake) should return error") + } +} + +func TestCollectTranscriptBlobHashes_SingleSession(t *testing.T) { + t.Parallel() + + _, store, cpID := setupRepoForUpdate(t) + + tree, err := store.getSessionsBranchTree() + if err != nil { + t.Fatalf("getSessionsBranchTree() error = %v", err) + } + + refs, err := CollectTranscriptBlobHashes(tree, cpID) + if err != nil { + t.Fatalf("CollectTranscriptBlobHashes() error = %v", err) + } + + if len(refs) != 1 { + t.Fatalf("expected 1 transcript ref, got %d", len(refs)) + } + + ref := refs[0] + if ref.SessionIndex != 0 { + t.Errorf("SessionIndex = %d, want 0", ref.SessionIndex) + } + if ref.Hash.IsZero() { + t.Error("Hash should not be zero") + } + if ref.Path != "0/full.jsonl" { + t.Errorf("Path = %q, want %q", ref.Path, "0/full.jsonl") + } +} + +func TestCollectTranscriptBlobHashes_MultiSession(t *testing.T) { + t.Parallel() + + repo, store, cpID := setupRepoForUpdate(t) + + // Write a second session to the same checkpoint + err := store.WriteCommitted(context.Background(), WriteCommittedOptions{ + CheckpointID: cpID, + SessionID: "session-002", + Strategy: "manual-commit", + Transcript: []byte("second session transcript\n"), + Prompts: []string{"second prompt"}, + AuthorName: "Test", + AuthorEmail: "test@test.com", + }) + if err != nil { + t.Fatalf("WriteCommitted() for second session error = %v", err) + } + + tree, err := store.getSessionsBranchTree() + if err != nil { + t.Fatalf("getSessionsBranchTree() error = %v", err) + } + + refs, err := CollectTranscriptBlobHashes(tree, cpID) + if err != nil { + t.Fatalf("CollectTranscriptBlobHashes() error = %v", err) + } + + if len(refs) != 2 { + t.Fatalf("expected 2 transcript refs, got %d", len(refs)) + } + + // Verify session indices + if refs[0].SessionIndex != 0 { + t.Errorf("refs[0].SessionIndex = %d, want 0", refs[0].SessionIndex) + } + if refs[1].SessionIndex != 1 { + t.Errorf("refs[1].SessionIndex = %d, want 1", refs[1].SessionIndex) + } + + // Verify they have different hashes (different transcript content) + if refs[0].Hash == refs[1].Hash { + t.Error("multi-session refs should have different blob hashes") + } + + // Verify all blobs exist locally + resolver := NewBlobResolver(repo.Storer) + for i, ref := range refs { + if !resolver.HasBlob(ref.Hash) { + t.Errorf("session %d blob %s should be present locally", i, ref.Hash) + } + } +} + +func TestCollectTranscriptBlobHashes_NonexistentCheckpoint(t *testing.T) { + t.Parallel() + + _, store, _ := setupRepoForUpdate(t) + + tree, err := store.getSessionsBranchTree() + if err != nil { + t.Fatalf("getSessionsBranchTree() error = %v", err) + } + + fakeID := id.MustCheckpointID("ffffffffffff") + _, err = CollectTranscriptBlobHashes(tree, fakeID) + if err == nil { + t.Error("expected error for nonexistent checkpoint") + } +} diff --git a/cmd/entire/cli/checkpoint/committed.go b/cmd/entire/cli/checkpoint/committed.go index c660b0a132..47a75beb87 100644 --- a/cmd/entire/cli/checkpoint/committed.go +++ b/cmd/entire/cli/checkpoint/committed.go @@ -740,18 +740,18 @@ func (s *GitStore) ReadCommitted(ctx context.Context, checkpointID id.Checkpoint return nil, err //nolint:wrapcheck // Propagating context cancellation } - tree, err := s.getSessionsBranchTree() + ft, err := s.getFetchingTree(ctx) if err != nil { return nil, nil //nolint:nilnil,nilerr // No sessions branch means no checkpoint exists } checkpointPath := checkpointID.Path() - checkpointTree, err := tree.Tree(checkpointPath) + checkpointTree, err := ft.Tree(checkpointPath) if err != nil { return nil, nil //nolint:nilnil,nilerr // Checkpoint directory not found } - // Read root metadata.json as CheckpointSummary + // Read root metadata.json as CheckpointSummary (auto-fetches blob if needed) metadataFile, err := checkpointTree.File(paths.MetadataFileName) if err != nil { return nil, nil //nolint:nilnil,nilerr // metadata.json not found @@ -779,13 +779,13 @@ func (s *GitStore) ReadSessionContent(ctx context.Context, checkpointID id.Check return nil, err //nolint:wrapcheck // Propagating context cancellation } - tree, err := s.getSessionsBranchTree() + ft, err := s.getFetchingTree(ctx) if err != nil { return nil, ErrCheckpointNotFound } checkpointPath := checkpointID.Path() - checkpointTree, err := tree.Tree(checkpointPath) + checkpointTree, err := ft.Tree(checkpointPath) if err != nil { return nil, ErrCheckpointNotFound } @@ -799,7 +799,7 @@ func (s *GitStore) ReadSessionContent(ctx context.Context, checkpointID id.Check result := &SessionContent{} - // Read session-specific metadata + // Read session-specific metadata (auto-fetches blob if needed) var agentType types.AgentType if metadataFile, fileErr := sessionTree.File(paths.MetadataFileName); fileErr == nil { if content, contentErr := metadataFile.Contents(); contentErr == nil { @@ -809,12 +809,12 @@ func (s *GitStore) ReadSessionContent(ctx context.Context, checkpointID id.Check } } - // Read transcript + // Read transcript (auto-fetches blobs if needed) if transcript, transcriptErr := readTranscriptFromTree(ctx, sessionTree, agentType); transcriptErr == nil && transcript != nil { result.Transcript = transcript } - // Read prompts + // Read prompts (auto-fetches blob if needed) if file, fileErr := sessionTree.File(paths.PromptFileName); fileErr == nil { if content, contentErr := file.Contents(); contentErr == nil { result.Prompts = content @@ -1295,6 +1295,17 @@ func (s *GitStore) ensureSessionsBranch() error { return nil } +// getFetchingTree returns a FetchingTree for the metadata branch. +// If a blob fetcher is configured on the store, File() calls on the returned +// tree will automatically fetch missing blobs from the remote. +func (s *GitStore) getFetchingTree(ctx context.Context) (*FetchingTree, error) { + tree, err := s.getSessionsBranchTree() + if err != nil { + return nil, err + } + return NewFetchingTree(ctx, tree, s.repo.Storer, s.blobFetcher), nil +} + // getSessionsBranchTree returns the tree object for the entire/checkpoints/v1 branch. // Falls back to origin/entire/checkpoints/v1 if the local branch doesn't exist. func (s *GitStore) getSessionsBranchTree() (*object.Tree, error) { @@ -1530,12 +1541,12 @@ func CreateCommit(repo *git.Repository, treeHash, parentHash plumbing.Hash, mess // readTranscriptFromTree reads a transcript from a git tree, handling both chunked and non-chunked formats. // It checks for chunk files first (.001, .002, etc.), then falls back to the base file. // The agentType is used for reassembling chunks in the correct format. -func readTranscriptFromTree(ctx context.Context, tree *object.Tree, agentType types.AgentType) ([]byte, error) { +func readTranscriptFromTree(ctx context.Context, tree *FetchingTree, agentType types.AgentType) ([]byte, error) { // Collect all transcript-related files var chunkFiles []string var hasBaseFile bool - for _, entry := range tree.Entries { + for _, entry := range tree.RawEntries() { if entry.Name == paths.TranscriptFileName || entry.Name == paths.TranscriptFileNameLegacy { hasBaseFile = true } diff --git a/cmd/entire/cli/checkpoint/fetching_tree.go b/cmd/entire/cli/checkpoint/fetching_tree.go new file mode 100644 index 0000000000..845ca6f043 --- /dev/null +++ b/cmd/entire/cli/checkpoint/fetching_tree.go @@ -0,0 +1,243 @@ +package checkpoint + +import ( + "context" + "fmt" + "io" + "log/slog" + "os/exec" + + "github.com/entireio/cli/cmd/entire/cli/logging" + + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/plumbing/storer" +) + +// BlobFetchFunc fetches missing blob objects by hash from a remote. +type BlobFetchFunc func(ctx context.Context, hashes []plumbing.Hash) error + +// FetchingTree wraps a git tree to automatically fetch missing blobs on demand. +// After a treeless fetch (--filter=blob:none), tree objects are available locally +// but blob objects are not. Each File() call checks whether the target blob +// exists locally and fetches it from the remote if missing, using FindEntry +// to locate the blob hash without resolving the blob itself. +// +// Because go-git's ObjectStorage caches the packfile index and never refreshes +// it, blobs fetched by external git commands (e.g. git fetch-pack) may not be +// visible to go-git's storer. As a fallback, File() reads the blob via +// "git cat-file" which always sees the current on-disk object store. +// +// For best performance, call PreFetch before reading files. PreFetch walks +// the tree, identifies locally-missing blobs, and batch-fetches them in a +// single network round-trip instead of one fetch per File() miss. +type FetchingTree struct { + inner *object.Tree + ctx context.Context + storer storer.EncodedObjectStorer + fetch BlobFetchFunc +} + +// NewFetchingTree wraps a git tree with on-demand blob fetching. +// The storer is used to check if blobs exist locally, and fetch is called +// to download any that are missing. If fetch is nil, File() behaves +// identically to the underlying tree. +func NewFetchingTree(ctx context.Context, tree *object.Tree, s storer.EncodedObjectStorer, fetch BlobFetchFunc) *FetchingTree { + return &FetchingTree{ + inner: tree, + ctx: ctx, + storer: s, + fetch: fetch, + } +} + +// File returns the file at the given path. If the blob is not available +// locally (e.g. after a treeless fetch), it is fetched on demand. If go-git's +// storer still can't see the blob after fetching (due to cached packfile index), +// the blob is read via "git cat-file" and an in-memory File is returned. +func (t *FetchingTree) File(path string) (*object.File, error) { + // Fast path: blob already available in go-git's storer. + file, err := t.inner.File(path) + if err == nil { + return file, nil + } + + if t.fetch == nil { + return nil, err //nolint:wrapcheck // pass-through wrapper + } + + // Find the tree entry to get the blob hash without resolving the blob. + // FindEntry only navigates tree objects (available after --filter=blob:none). + entry, findErr := t.inner.FindEntry(path) + if findErr != nil { + logging.Debug(t.ctx, "FetchingTree.File: entry not found", + slog.String("path", path), + slog.String("error", findErr.Error()), + ) + return nil, err //nolint:wrapcheck // return original File() error + } + + logging.Debug(t.ctx, "FetchingTree.File: blob missing, fetching", + slog.String("path", path), + slog.String("hash", entry.Hash.String()[:12]), + ) + + // Fetch the blob from the remote. + if fetchErr := t.fetch(t.ctx, []plumbing.Hash{entry.Hash}); fetchErr != nil { + logging.Warn(t.ctx, "FetchingTree.File: blob fetch failed", + slog.String("path", path), + slog.String("hash", entry.Hash.String()[:12]), + slog.String("error", fetchErr.Error()), + ) + return nil, err //nolint:wrapcheck // return original File() error + } + + // Try go-git again — works if blob was stored as a loose object. + file, err = t.inner.File(path) + if err == nil { + return file, nil + } + + // go-git's storer caches the packfile index and won't see new packs + // created by external git commands. Fall back to "git cat-file" which + // reads directly from the on-disk object store. + logging.Debug(t.ctx, "FetchingTree.File: storer cache stale, reading via git cat-file", + slog.String("path", path), + slog.String("hash", entry.Hash.String()[:12]), + ) + return t.readFileViaGit(path, entry) +} + +// PreFetch walks the tree recursively, identifies blob entries that are missing +// from the local object store, and batch-fetches them in a single call to +// t.fetch. This avoids per-blob network round-trips during subsequent File() +// calls. It is safe to call even when all blobs are already local (no-op). +// Returns the number of blobs fetched. +func (t *FetchingTree) PreFetch() (int, error) { + if t.fetch == nil || t.storer == nil { + return 0, nil + } + + missing := t.collectMissingBlobs(t.inner) + if len(missing) == 0 { + return 0, nil + } + + logging.Debug(t.ctx, "FetchingTree.PreFetch: batch-fetching missing blobs", + slog.Int("count", len(missing)), + ) + + if err := t.fetch(t.ctx, missing); err != nil { + return 0, fmt.Errorf("prefetch %d blobs: %w", len(missing), err) + } + + return len(missing), nil +} + +// collectMissingBlobs recursively walks a tree and returns hashes of blob +// entries that are not present in the local object store. +func (t *FetchingTree) collectMissingBlobs(tree *object.Tree) []plumbing.Hash { + var missing []plumbing.Hash + for _, entry := range tree.Entries { + if entry.Mode.IsFile() { + if t.storer.HasEncodedObject(entry.Hash) != nil { + missing = append(missing, entry.Hash) + } + } else { + // Recurse into subtrees (tree objects are local after treeless fetch). + subtree, err := tree.Tree(entry.Name) + if err == nil { + missing = append(missing, t.collectMissingBlobs(subtree)...) + } + } + } + return missing +} + +// readFileViaGit reads a blob via "git cat-file -p " and returns an +// in-memory *object.File. This bypasses go-git's storer which may have a +// stale packfile index after external git commands fetched new objects. +func (t *FetchingTree) readFileViaGit(path string, entry *object.TreeEntry) (*object.File, error) { + cmd := exec.CommandContext(t.ctx, "git", "cat-file", "-p", entry.Hash.String()) + content, cmdErr := cmd.Output() + if cmdErr != nil { + logging.Warn(t.ctx, "FetchingTree.readFileViaGit: cat-file failed", + slog.String("path", path), + slog.String("hash", entry.Hash.String()[:12]), + slog.String("error", cmdErr.Error()), + ) + return nil, fmt.Errorf("blob %s not readable after fetch: %w", entry.Hash.String()[:12], cmdErr) + } + + // Create an in-memory encoded object to construct the File. + memObj := &plumbing.MemoryObject{} + memObj.SetType(plumbing.BlobObject) + memObj.SetSize(int64(len(content))) + w, wErr := memObj.Writer() + if wErr != nil { + return nil, fmt.Errorf("memory object writer: %w", wErr) + } + if _, wErr = w.Write(content); wErr != nil { + return nil, fmt.Errorf("memory object write: %w", wErr) + } + if wErr = w.Close(); wErr != nil { + return nil, fmt.Errorf("memory object close: %w", wErr) + } + + blob := &object.Blob{} + if dErr := blob.Decode(memObj); dErr != nil { + return nil, fmt.Errorf("blob decode: %w", dErr) + } + + logging.Debug(t.ctx, "FetchingTree.readFileViaGit: blob read successfully", + slog.String("path", path), + slog.String("hash", entry.Hash.String()[:12]), + slog.Int64("size", int64(len(content))), + ) + + return object.NewFile(path, entry.Mode, blob), nil +} + +// Tree returns the subtree at the given path, wrapped with the same fetching +// behavior. +func (t *FetchingTree) Tree(path string) (*FetchingTree, error) { + subtree, err := t.inner.Tree(path) + if err != nil { + return nil, fmt.Errorf("tree %s: %w", path, err) + } + return &FetchingTree{ + inner: subtree, + ctx: t.ctx, + storer: t.storer, + fetch: t.fetch, + }, nil +} + +// RawEntries returns the direct tree entries (no blob reads needed). +func (t *FetchingTree) RawEntries() []object.TreeEntry { + return t.inner.Entries +} + +// Unwrap returns the underlying *object.Tree. +func (t *FetchingTree) Unwrap() *object.Tree { + return t.inner +} + +// Files returns a recursive file iterator from the underlying tree. +// Warning: after a treeless fetch, this iterator will fail when it tries +// to resolve blob objects. Use File() for on-demand blob fetching instead. +func (t *FetchingTree) Files() *object.FileIter { + return t.inner.Files() +} + +// FileReader provides read access to files within a git tree. +// Both *object.Tree and *FetchingTree implement this interface. +type FileReader interface { + File(path string) (*object.File, error) +} + +// FileOpener provides access to a file's content reader. +// *object.File implements this interface. +type FileOpener interface { + Reader() (io.ReadCloser, error) +} diff --git a/cmd/entire/cli/checkpoint/store.go b/cmd/entire/cli/checkpoint/store.go index 283dca51c4..2eac012c86 100644 --- a/cmd/entire/cli/checkpoint/store.go +++ b/cmd/entire/cli/checkpoint/store.go @@ -10,7 +10,8 @@ var _ Store = (*GitStore)(nil) // GitStore provides operations for both temporary and committed checkpoint storage. // It implements the Store interface by wrapping a git repository. type GitStore struct { - repo *git.Repository + repo *git.Repository + blobFetcher BlobFetchFunc } // NewGitStore creates a new checkpoint store backed by the given git repository. @@ -18,6 +19,13 @@ func NewGitStore(repo *git.Repository) *GitStore { return &GitStore{repo: repo} } +// SetBlobFetcher configures the store to automatically fetch missing blobs +// on demand when reading from metadata trees. This is used after treeless +// fetches where tree objects are local but blob objects are not. +func (s *GitStore) SetBlobFetcher(f BlobFetchFunc) { + s.blobFetcher = f +} + // Repository returns the underlying git repository. // This is useful for strategies that need direct repository access. func (s *GitStore) Repository() *git.Repository { diff --git a/cmd/entire/cli/checkpoint/temporary.go b/cmd/entire/cli/checkpoint/temporary.go index dc77a6b3f2..b9442fe324 100644 --- a/cmd/entire/cli/checkpoint/temporary.go +++ b/cmd/entire/cli/checkpoint/temporary.go @@ -586,8 +586,10 @@ func (s *GitStore) GetTranscriptFromCommit(ctx context.Context, commitHash plumb // Try to get the metadata subtree for chunk detection subTree, subTreeErr := tree.Tree(metadataDir) if subTreeErr == nil { - // Use the helper function that handles chunking - transcript, err := readTranscriptFromTree(ctx, subTree, agentType) + // Use the helper function that handles chunking. + // Wrap in FetchingTree with nil fetcher (temporary reads are always local). + ft := &FetchingTree{inner: subTree} + transcript, err := readTranscriptFromTree(ctx, ft, agentType) if err == nil && transcript != nil { return transcript, nil } diff --git a/cmd/entire/cli/config.go b/cmd/entire/cli/config.go index bd5020c48d..72cb02b21c 100644 --- a/cmd/entire/cli/config.go +++ b/cmd/entire/cli/config.go @@ -62,9 +62,12 @@ func IsEnabled(ctx context.Context) (bool, error) { return s.Enabled, nil } -// GetStrategy returns the manual-commit strategy instance. +// GetStrategy returns the manual-commit strategy instance with blob fetching +// enabled so that checkpoint reads work after treeless fetches. func GetStrategy(_ context.Context) *strategy.ManualCommitStrategy { - return strategy.NewManualCommitStrategy() + s := strategy.NewManualCommitStrategy() + s.SetBlobFetcher(FetchBlobsByHash) + return s } // GetLogLevel returns the configured log level from settings. diff --git a/cmd/entire/cli/config_test.go b/cmd/entire/cli/config_test.go index 1ff70d596a..f8143b586c 100644 --- a/cmd/entire/cli/config_test.go +++ b/cmd/entire/cli/config_test.go @@ -284,6 +284,25 @@ func TestLoadEntireSettings_RejectsUnknownKeysInBase(t *testing.T) { } } +// TestGetStrategy_HasBlobFetcher verifies that GetStrategy returns a strategy +// with blob fetching enabled. Without this, checkpoint reads after a treeless +// fetch (--filter=blob:none) silently fail because blobs are not local and +// FetchingTree has no fetcher to download them — causing "session log not +// available" errors during resume. +// +// Regression test for the bug introduced in b92b37b3 where ReadCommitted and +// ReadSessionContent were changed to use FetchingTree but GetStrategy did not +// configure a blob fetcher on the strategy. +func TestGetStrategy_HasBlobFetcher(t *testing.T) { + t.Parallel() + + strat := GetStrategy(context.Background()) + if !strat.HasBlobFetcher() { + t.Fatal("GetStrategy must return a strategy with a blob fetcher configured; " + + "without it, checkpoint reads fail after treeless fetches") + } +} + func TestLoadEntireSettings_RejectsUnknownKeysInLocal(t *testing.T) { setupLocalOverrideTestDir(t) diff --git a/cmd/entire/cli/git_operations.go b/cmd/entire/cli/git_operations.go index cec82c1c13..cc555172ed 100644 --- a/cmd/entire/cli/git_operations.go +++ b/cmd/entire/cli/git_operations.go @@ -1,13 +1,16 @@ package cli import ( + "bytes" "context" "errors" "fmt" + "log/slog" "os/exec" "strings" "time" + "github.com/entireio/cli/cmd/entire/cli/logging" "github.com/entireio/cli/cmd/entire/cli/paths" "github.com/entireio/cli/cmd/entire/cli/strategy" @@ -246,7 +249,9 @@ func findNewUntrackedFiles(current, preExisting []string) []string { } // BranchExistsOnRemote checks if a branch exists on the origin remote. -// Returns true if the branch is tracked on origin, false otherwise. +// First checks local remote-tracking refs, then queries the actual remote +// via git ls-remote in case local refs are stale (e.g., after a fresh clone +// that didn't fetch all branches). func BranchExistsOnRemote(ctx context.Context, branchName string) (bool, error) { repo, err := openRepository(ctx) if err != nil { @@ -255,14 +260,25 @@ func BranchExistsOnRemote(ctx context.Context, branchName string) (bool, error) // Check for remote reference: refs/remotes/origin/ _, err = repo.Reference(plumbing.NewRemoteReferenceName("origin", branchName), true) - if err != nil { - if errors.Is(err, plumbing.ErrReferenceNotFound) { - return false, nil - } + if err == nil { + return true, nil + } + if !errors.Is(err, plumbing.ErrReferenceNotFound) { return false, fmt.Errorf("failed to check remote branch: %w", err) } - return true, nil + // Local remote-tracking ref not found — query the actual remote. + lsCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + lsCmd := exec.CommandContext(lsCtx, "git", "ls-remote", "--heads", "origin", "refs/heads/"+branchName) + output, lsErr := lsCmd.Output() + if lsErr != nil { + // ls-remote failed (no network, no remote, etc.) — treat as not found + return false, nil + } + + return len(bytes.TrimSpace(output)) > 0, nil } // BranchExistsLocally checks if a local branch exists. @@ -394,3 +410,82 @@ func FetchMetadataBranch(ctx context.Context) error { return nil } + +// FetchMetadataTreeOnly fetches the tip of the entire/checkpoints/v1 branch +// from origin with --depth=1 --filter=blob:none, downloading only the latest +// commit and its tree objects (no blobs, no history). +// After this call, tree navigation via go-git works but blob reads will fail +// for objects that weren't previously fetched. +// Uses git CLI for credential helper support. +func FetchMetadataTreeOnly(ctx context.Context) error { + branchName := paths.MetadataBranchName + + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + refSpec := fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", branchName, branchName) + + fetchCmd := exec.CommandContext(ctx, "git", "fetch", "--depth=1", "--filter=blob:none", "origin", refSpec) + if output, err := fetchCmd.CombinedOutput(); err != nil { + if ctx.Err() == context.DeadlineExceeded { + return errors.New("treeless fetch timed out after 2 minutes") + } + return fmt.Errorf("failed to treeless-fetch %s from origin: %s: %w", branchName, strings.TrimSpace(string(output)), err) + } + + repo, err := openRepository(ctx) + if err != nil { + return fmt.Errorf("failed to open repository: %w", err) + } + + // Get the remote branch reference + remoteRef, err := repo.Reference(plumbing.NewRemoteReferenceName("origin", branchName), true) + if err != nil { + return fmt.Errorf("branch '%s' not found on origin: %w", branchName, err) + } + + // Create or update local branch pointing to the same commit + localRef := plumbing.NewHashReference(plumbing.NewBranchReferenceName(branchName), remoteRef.Hash()) + if err := repo.Storer.SetReference(localRef); err != nil { + return fmt.Errorf("failed to create local %s branch: %w", branchName, err) + } + + return nil +} + +// FetchBlobsByHash fetches specific blob objects from the remote by their SHA-1 hashes. +// Uses "git fetch origin " which goes through normal credential helpers, +// unlike fetch-pack which bypasses them. Requires the server to support +// uploadpack.allowReachableSHA1InWant (GitHub, GitLab, Bitbucket all do). +// +// If fetching by hash fails, falls back to a full metadata branch fetch. +func FetchBlobsByHash(ctx context.Context, hashes []plumbing.Hash) error { + if len(hashes) == 0 { + return nil + } + + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + // Build fetch args: "git fetch origin ..." + // This uses the normal transport + credential helpers, unlike fetch-pack. + args := []string{"fetch", "--no-write-fetch-head", "origin"} + for _, h := range hashes { + args = append(args, h.String()) + } + + fetchCmd := exec.CommandContext(ctx, "git", args...) + if _, fetchErr := fetchCmd.CombinedOutput(); fetchErr != nil { + logging.Debug(ctx, "fetch-by-hash failed, falling back to full metadata fetch", + slog.Int("blob_count", len(hashes)), + slog.String("error", fetchErr.Error()), + ) + // Fallback: full metadata branch fetch (pack negotiation skips already-local objects) + if fallbackErr := FetchMetadataBranch(ctx); fallbackErr != nil { + return fmt.Errorf("fetch-by-hash failed: %w; fallback fetch also failed: %w", + fetchErr, fallbackErr) + } + } + + return nil +} diff --git a/cmd/entire/cli/integration_test/testenv.go b/cmd/entire/cli/integration_test/testenv.go index 42eb2d28cd..c2fee1e626 100644 --- a/cmd/entire/cli/integration_test/testenv.go +++ b/cmd/entire/cli/integration_test/testenv.go @@ -218,6 +218,9 @@ func (env *TestEnv) InitRepo() { } cfg.Raw.Section("commit").SetOption("gpgsign", "false") + // Override any global core.hooksPath so tests use the repo-local hooks directory. + cfg.Raw.Section("core").SetOption("hooksPath", filepath.Join(env.RepoDir, ".git", "hooks")) + if err := repo.SetConfig(cfg); err != nil { env.T.Fatalf("failed to set repo config: %v", err) } diff --git a/cmd/entire/cli/resume.go b/cmd/entire/cli/resume.go index a78debeb1d..c43830e540 100644 --- a/cmd/entire/cli/resume.go +++ b/cmd/entire/cli/resume.go @@ -125,6 +125,8 @@ func runResume(ctx context.Context, cmd *cobra.Command, branchName string, force } func resumeFromCurrentBranch(ctx context.Context, w, errW io.Writer, branchName string, force bool) error { + logCtx := logging.WithComponent(ctx, "resume") + repo, err := openRepository(ctx) if err != nil { return fmt.Errorf("not a git repository: %w", err) @@ -140,6 +142,13 @@ func resumeFromCurrentBranch(ctx context.Context, w, errW io.Writer, branchName return nil } + logging.Debug(logCtx, "found checkpoint(s) on branch", + slog.String("branch", branchName), + slog.Int("checkpoint_count", len(result.checkpointIDs)), + slog.String("commit", result.commitHash[:7]), + slog.Bool("newer_commits_exist", result.newerCommitsExist), + ) + // If there are newer commits without checkpoints, ask for confirmation. // Merge commits (e.g., from merging main) don't count as "work" and are skipped silently. if result.newerCommitsExist && !force { @@ -163,87 +172,259 @@ func resumeFromCurrentBranch(ctx context.Context, w, errW io.Writer, branchName // resolveLatestCheckpoint also returns the metadata tree so we can reuse it // for the ReadCheckpointMetadata call below without a redundant lookup. var metadataTree *object.Tree + var freshRepo *git.Repository if len(result.checkpointIDs) > 1 { - latest, tree, err := resolveLatestCheckpoint(ctx, repo, result.checkpointIDs) + latest, tree, latestRepo, err := resolveLatestCheckpoint(ctx, result.checkpointIDs) if err != nil { // No metadata available — nothing to resume from + logging.Warn(logCtx, "resolveLatestCheckpoint failed", + slog.Int("checkpoint_count", len(result.checkpointIDs)), + slog.String("error", err.Error()), + ) fmt.Fprintf(w, "Found %d checkpoints for commit %s but metadata is not available\n", len(result.checkpointIDs), result.commitHash[:7]) - return checkRemoteMetadata(ctx, w, errW, repo, result.checkpointIDs[0]) + return checkRemoteMetadata(ctx, result.checkpointIDs[0]) } skipped := len(result.checkpointIDs) - 1 fmt.Fprintf(w, "Found %d checkpoints for commit %s, resuming from the latest (%d older checkpoints skipped)\n", len(result.checkpointIDs), result.commitHash[:7], skipped) checkpointID = latest metadataTree = tree + freshRepo = latestRepo } // Get metadata branch tree for lookups (reuse from resolveLatestCheckpoint if available) if metadataTree == nil { - var err error - metadataTree, err = strategy.GetMetadataBranchTree(repo) - if err != nil { - // No local metadata branch, check if remote has it - return checkRemoteMetadata(ctx, w, errW, repo, checkpointID) + var treeErr error + metadataTree, freshRepo, treeErr = getMetadataTree(ctx) + if treeErr != nil { + logging.Warn(logCtx, "getMetadataTree failed, checking remote", + slog.String("checkpoint_id", checkpointID.String()), + slog.String("error", treeErr.Error()), + ) + // All fetch attempts failed, check if remote has it + return checkRemoteMetadata(ctx, checkpointID) } } - // Look up metadata from sharded path - metadata, err := strategy.ReadCheckpointMetadata(metadataTree, checkpointID.Path()) + logging.Debug(logCtx, "metadata tree obtained", + slog.String("checkpoint_id", checkpointID.String()), + slog.String("checkpoint_path", checkpointID.Path()), + slog.String("tree_hash", metadataTree.Hash.String()), + ) + + // Navigate to the checkpoint subtree first (uses tree objects only, no blobs). + // This scopes the FetchingTree to only this checkpoint's files instead of + // the entire metadata branch. + cpSubtree, cpErr := metadataTree.Tree(checkpointID.Path()) + if cpErr != nil { + logging.Warn(logCtx, "checkpoint subtree not found in metadata tree", + slog.String("checkpoint_id", checkpointID.String()), + slog.String("checkpoint_path", checkpointID.Path()), + slog.String("tree_hash", metadataTree.Hash.String()), + slog.String("error", cpErr.Error()), + ) + return checkRemoteMetadata(ctx, checkpointID) + } + + // Log subtree details for diagnostics + var subtreeEntryNames []string + for _, e := range cpSubtree.Entries { + subtreeEntryNames = append(subtreeEntryNames, fmt.Sprintf("%s(%s:%s)", e.Name, e.Mode, e.Hash.String()[:7])) + } + logging.Debug(logCtx, "checkpoint subtree found", + slog.String("checkpoint_id", checkpointID.String()), + slog.String("subtree_hash", cpSubtree.Hash.String()), + slog.Int("entry_count", len(cpSubtree.Entries)), + slog.Any("entries", subtreeEntryNames), + ) + + // Wrap the checkpoint subtree with on-demand blob fetching. + // Use the fresh repo's storer (not the original repo) because a fetch may have + // created new packfiles that the original repo's storer doesn't know about. + ft := checkpoint.NewFetchingTree(ctx, cpSubtree, freshRepo.Storer, FetchBlobsByHash) + + // Batch-prefetch all missing blobs in one network round-trip instead of + // fetching one blob per File() call during metadata reads. + if prefetched, pfErr := ft.PreFetch(); pfErr != nil { + logging.Warn(logCtx, "PreFetch failed, falling back to per-blob fetching", + slog.String("checkpoint_id", checkpointID.String()), + slog.String("error", pfErr.Error()), + ) + } else if prefetched > 0 { + logging.Debug(logCtx, "PreFetch completed", + slog.String("checkpoint_id", checkpointID.String()), + slog.Int("blobs_fetched", prefetched), + ) + } + + // Read metadata from checkpoint subtree (paths are relative to checkpoint root) + metadata, err := strategy.ReadCheckpointMetadataFromSubtree(ft, checkpointID.Path()) if err != nil { - // Checkpoint exists in commit but no local metadata - check remote - return checkRemoteMetadata(ctx, w, errW, repo, checkpointID) + logging.Warn(logCtx, "ReadCheckpointMetadataFromSubtree failed, checking remote", + slog.String("checkpoint_id", checkpointID.String()), + slog.String("subtree_hash", cpSubtree.Hash.String()), + slog.String("error", err.Error()), + ) + return checkRemoteMetadata(ctx, checkpointID) } + logging.Debug(logCtx, "checkpoint metadata read successfully", + slog.String("checkpoint_id", checkpointID.String()), + slog.String("session_id", metadata.SessionID), + slog.Int("session_count", metadata.SessionCount), + ) + return resumeSession(ctx, w, errW, metadata, force) } // resolveLatestCheckpoint reads metadata for each checkpoint ID and returns -// the one with the latest CreatedAt, along with the metadata tree for reuse. -// It tries the local metadata branch first, then fetches from remote, then -// falls back to the remote tree directly. -func resolveLatestCheckpoint(ctx context.Context, repo *git.Repository, checkpointIDs []id.CheckpointID) (id.CheckpointID, *object.Tree, error) { - metadataTree, err := getMetadataTree(ctx, repo) +// the one with the latest CreatedAt, along with the metadata tree and fresh +// repo for reuse. It tries the local metadata branch first, then fetches from +// remote, then falls back to the remote tree directly. +func resolveLatestCheckpoint(ctx context.Context, checkpointIDs []id.CheckpointID) (id.CheckpointID, *object.Tree, *git.Repository, error) { + metadataTree, freshRepo, err := getMetadataTree(ctx) if err != nil { - return id.EmptyCheckpointID, nil, err + return id.EmptyCheckpointID, nil, nil, err } + infoMap := make(map[id.CheckpointID]strategy.CheckpointInfo, len(checkpointIDs)) for _, cpID := range checkpointIDs { - metadata, err := strategy.ReadCheckpointMetadata(metadataTree, cpID.Path()) - if err != nil { + // Navigate to each checkpoint's subtree, wrap with blob fetching + cpSubtree, cpErr := metadataTree.Tree(cpID.Path()) + if cpErr != nil { + logging.Debug(ctx, "resolveLatestCheckpoint: checkpoint subtree not found", + slog.String("checkpoint_id", cpID.String()), + slog.String("error", cpErr.Error()), + ) + continue + } + ft := checkpoint.NewFetchingTree(ctx, cpSubtree, freshRepo.Storer, FetchBlobsByHash) + // Batch-prefetch blobs for this checkpoint subtree. + if _, pfErr := ft.PreFetch(); pfErr != nil { + logging.Debug(ctx, "resolveLatestCheckpoint: PreFetch failed", + slog.String("checkpoint_id", cpID.String()), + slog.String("error", pfErr.Error()), + ) + } + metadata, metaErr := strategy.ReadCheckpointMetadataFromSubtree(ft, cpID.Path()) + if metaErr != nil { + logging.Debug(ctx, "resolveLatestCheckpoint: checkpoint metadata read failed", + slog.String("checkpoint_id", cpID.String()), + slog.String("error", metaErr.Error()), + ) continue } infoMap[cpID] = *metadata } latest, found := strategy.ResolveLatestCheckpointFromMap(checkpointIDs, infoMap) if !found { - return id.EmptyCheckpointID, nil, errors.New("no checkpoint metadata found") + return id.EmptyCheckpointID, nil, nil, errors.New("no checkpoint metadata found") } - return latest.CheckpointID, metadataTree, nil + return latest.CheckpointID, metadataTree, freshRepo, nil } -// getMetadataTree returns the metadata branch tree, trying local first, -// then fetching from remote, then falling back to the remote tree directly. -func getMetadataTree(ctx context.Context, repo *git.Repository) (*object.Tree, error) { - metadataTree, err := strategy.GetMetadataBranchTree(repo) - if err == nil { - return metadataTree, nil +// getMetadataTree returns the metadata branch tree and a fresh repo handle. +// After a fetch, go-git's storer cache may be stale (new packfiles on disk +// are invisible to the repo opened before the fetch). To avoid this, each +// attempt opens a fresh repo after the fetch succeeds. +// +// Fallback order: treeless fetch → local → full fetch → remote tree. +func getMetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) { + logCtx := logging.WithComponent(ctx, "resume.getMetadataTree") + + // Helper to log ref hash for a repo's metadata branch + logRefHash := func(repo *git.Repository, source string) { + ref, refErr := repo.Reference(plumbing.NewBranchReferenceName("entire/checkpoints/v1"), true) + if refErr != nil { + logging.Debug(logCtx, "metadata branch ref not found", + slog.String("source", source), + slog.String("error", refErr.Error()), + ) + return + } + logging.Debug(logCtx, "metadata branch ref resolved", + slog.String("source", source), + slog.String("ref_hash", ref.Hash().String()), + ) } - // Try fetching from remote - if fetchErr := FetchMetadataBranch(ctx); fetchErr == nil { - metadataTree, err = strategy.GetMetadataBranchTree(repo) + // Always try treeless fetch first to ensure local branch is up-to-date + if fetchErr := FetchMetadataTreeOnly(ctx); fetchErr == nil { + // Open a fresh repo so the storer sees new packfiles from the fetch + freshRepo, repoErr := openRepository(ctx) + if repoErr == nil { + logRefHash(freshRepo, "treeless-fetch") + metadataTree, treeErr := strategy.GetMetadataBranchTree(freshRepo) + if treeErr == nil { + logging.Debug(logCtx, "metadata tree obtained via treeless fetch", + slog.String("tree_hash", metadataTree.Hash.String()), + ) + return metadataTree, freshRepo, nil + } + logging.Debug(logCtx, "treeless fetch succeeded but tree read failed", + slog.String("error", treeErr.Error()), + ) + } + } else { + logging.Debug(logCtx, "treeless fetch failed, trying local", + slog.String("error", fetchErr.Error()), + ) + } + + // Try local (may have been set by a prior fetch or push) + localRepo, repoErr := openRepository(ctx) + if repoErr == nil { + logRefHash(localRepo, "local") + metadataTree, err := strategy.GetMetadataBranchTree(localRepo) if err == nil { - return metadataTree, nil + logging.Debug(logCtx, "metadata tree obtained from local branch", + slog.String("tree_hash", metadataTree.Hash.String()), + ) + return metadataTree, localRepo, nil } + logging.Debug(logCtx, "local metadata branch not available", + slog.String("error", err.Error()), + ) } - // Try remote tree directly - remoteTree, remoteErr := strategy.GetRemoteMetadataBranchTree(repo) + // Fallback: full fetch from remote + if fetchErr := FetchMetadataBranch(ctx); fetchErr == nil { + freshRepo, repoErr := openRepository(ctx) + if repoErr == nil { + logRefHash(freshRepo, "full-fetch") + metadataTree, treeErr := strategy.GetMetadataBranchTree(freshRepo) + if treeErr == nil { + logging.Debug(logCtx, "metadata tree obtained via full fetch", + slog.String("tree_hash", metadataTree.Hash.String()), + ) + return metadataTree, freshRepo, nil + } + logging.Debug(logCtx, "full fetch succeeded but tree read failed", + slog.String("error", treeErr.Error()), + ) + } + } else { + logging.Debug(logCtx, "full fetch failed", + slog.String("error", fetchErr.Error()), + ) + } + + // Try remote tree directly (origin/entire/checkpoints/v1) + remoteRepo, repoErr := openRepository(ctx) + if repoErr != nil { + return nil, nil, fmt.Errorf("failed to open repository: %w", repoErr) + } + logRefHash(remoteRepo, "remote-tracking") + remoteTree, remoteErr := strategy.GetRemoteMetadataBranchTree(remoteRepo) if remoteErr != nil { - return nil, fmt.Errorf("metadata branch not available: %w", remoteErr) + logging.Debug(logCtx, "remote metadata tree also not available", + slog.String("error", remoteErr.Error()), + ) + return nil, nil, fmt.Errorf("metadata branch not available: %w", remoteErr) } - return remoteTree, nil + logging.Debug(logCtx, "metadata tree obtained from remote-tracking branch") + return remoteTree, remoteRepo, nil } // branchCheckpointsResult contains the result of searching for checkpoints on a branch. @@ -399,32 +580,56 @@ func promptResumeFromOlderCheckpoint() (bool, error) { // checkRemoteMetadata checks if checkpoint metadata exists on origin/entire/checkpoints/v1 // and automatically fetches it if available. -func checkRemoteMetadata(ctx context.Context, w, errW io.Writer, repo *git.Repository, checkpointID id.CheckpointID) error { +func checkRemoteMetadata(ctx context.Context, checkpointID id.CheckpointID) error { + logCtx := logging.WithComponent(ctx, "resume.checkRemoteMetadata") + + // Open a fresh repo to avoid stale packfile index issues + repo, repoErr := openRepository(ctx) + if repoErr != nil { + logging.Warn(logCtx, "failed to open repository for remote check", + slog.String("error", repoErr.Error()), + ) + fmt.Fprintf(os.Stderr, "Checkpoint '%s' found in commit but session metadata not available\n", checkpointID) + return nil + } + // Try to get remote metadata branch tree remoteTree, err := strategy.GetRemoteMetadataBranchTree(repo) if err != nil { - fmt.Fprintf(w, "Checkpoint '%s' found in commit but session metadata not available\n", checkpointID) - fmt.Fprintf(w, "The entire/checkpoints/v1 branch may not exist locally or on the remote.\n") + fmt.Fprintf(os.Stderr, "Checkpoint '%s' found in commit but the entire/checkpoints/v1 branch is not available locally or on the remote.\n", checkpointID) + fmt.Fprintf(os.Stderr, "This can happen if the metadata branch was not pushed. Try:\n") + fmt.Fprintf(os.Stderr, " git fetch origin entire/checkpoints/v1:entire/checkpoints/v1\n") return nil //nolint:nilerr // Informational message, not a fatal error } - // Check if the checkpoint exists on the remote - metadata, err := strategy.ReadCheckpointMetadata(remoteTree, checkpointID.Path()) - if err != nil { - fmt.Fprintf(w, "Checkpoint '%s' found in commit but session metadata not available\n", checkpointID) + // Navigate to checkpoint subtree, then wrap with blob fetching + cpSubtree, cpErr := remoteTree.Tree(checkpointID.Path()) + if cpErr != nil { + fmt.Fprintf(os.Stderr, "Checkpoint '%s' found in commit but its metadata could not be read from entire/checkpoints/v1.\n", checkpointID) + fmt.Fprintf(os.Stderr, "The metadata branch exists but checkpoint content may be missing or unreadable (e.g. after a partial fetch).\n") + fmt.Fprintf(os.Stderr, "Try a full fetch:\n") + fmt.Fprintf(os.Stderr, " git fetch origin entire/checkpoints/v1:entire/checkpoints/v1\n") return nil //nolint:nilerr // Informational message, not a fatal error } - - // Metadata exists on remote but not locally - fetch it automatically - fmt.Fprintf(w, "Fetching session metadata from origin...\n") - if err := FetchMetadataBranch(ctx); err != nil { - fmt.Fprintf(errW, "Error: failed to fetch metadata: %v\n", err) - fmt.Fprintf(errW, "You can try manually: git fetch origin entire/checkpoints/v1:entire/checkpoints/v1\n") - return NewSilentError(errors.New("failed to fetch metadata")) + ft := checkpoint.NewFetchingTree(ctx, cpSubtree, repo.Storer, FetchBlobsByHash) + // Batch-prefetch blobs for the remote checkpoint subtree. + if _, pfErr := ft.PreFetch(); pfErr != nil { + logging.Debug(ctx, "checkRemoteMetadata: PreFetch failed", + slog.String("checkpoint_id", checkpointID.String()), + slog.String("error", pfErr.Error()), + ) + } + metadata, err := strategy.ReadCheckpointMetadataFromSubtree(ft, checkpointID.Path()) + if err != nil { + fmt.Fprintf(os.Stderr, "Checkpoint '%s' found in commit but its metadata could not be read from entire/checkpoints/v1.\n", checkpointID) + fmt.Fprintf(os.Stderr, "The metadata branch exists but checkpoint content may be missing or unreadable (e.g. after a partial fetch).\n") + fmt.Fprintf(os.Stderr, "Try a full fetch:\n") + fmt.Fprintf(os.Stderr, " git fetch origin entire/checkpoints/v1:entire/checkpoints/v1\n") + return nil //nolint:nilerr // Informational message, not a fatal error } - // Now resume the session with the fetched metadata - return resumeSession(ctx, w, errW, metadata, false) + // Metadata exists on remote — resume the session + return resumeSession(ctx, os.Stdout, os.Stderr, metadata, false) } // resumeSession restores and displays the resume command for a specific session. diff --git a/cmd/entire/cli/resume_test.go b/cmd/entire/cli/resume_test.go index 570130f117..69d4448069 100644 --- a/cmd/entire/cli/resume_test.go +++ b/cmd/entire/cli/resume_test.go @@ -3,7 +3,6 @@ package cli import ( "bytes" "context" - "errors" "fmt" "io" "os" @@ -474,7 +473,7 @@ func TestResolveLatestCheckpoint(t *testing.T) { // Pass checkpoint IDs in reverse chronological order (newest first), // simulating git CLI squash merge trailer order. reverseOrderIDs := []id.CheckpointID{cpID3, cpID2, cpID1} - latest, tree, err := resolveLatestCheckpoint(context.Background(), repo, reverseOrderIDs) + latest, tree, _, err := resolveLatestCheckpoint(context.Background(), reverseOrderIDs) if err != nil { t.Fatalf("resolveLatestCheckpoint() error = %v", err) } @@ -491,7 +490,7 @@ func TestResolveLatestCheckpoint(t *testing.T) { // Also verify with chronological order chronologicalIDs := []id.CheckpointID{cpID1, cpID2, cpID3} - latest2, _, err := resolveLatestCheckpoint(context.Background(), repo, chronologicalIDs) + latest2, _, _, err := resolveLatestCheckpoint(context.Background(), chronologicalIDs) if err != nil { t.Fatalf("resolveLatestCheckpoint() error = %v", err) } @@ -629,17 +628,13 @@ func TestCheckRemoteMetadata_MetadataExistsOnRemote(t *testing.T) { t.Fatalf("Failed to remove local metadata branch: %v", err) } - // Call checkRemoteMetadata - should find it on remote and attempt to fetch - // In this test environment without a real origin remote, the fetch will fail - // but it should return a SilentError (user-friendly error message already printed) - err = checkRemoteMetadata(context.Background(), io.Discard, io.Discard, repo, checkpointID) + // Call checkRemoteMetadata - should find metadata on the remote tree and + // attempt to resume, but fail because the test checkpoint has no agent field. + err = checkRemoteMetadata(context.Background(), checkpointID) if err == nil { - t.Error("checkRemoteMetadata() should return SilentError when fetch fails") - } else { - var silentErr *SilentError - if !errors.As(err, &silentErr) { - t.Errorf("checkRemoteMetadata() should return SilentError, got: %v", err) - } + t.Error("checkRemoteMetadata() should return error when agent is missing from metadata") + } else if !strings.Contains(err.Error(), "failed to resolve agent") { + t.Errorf("checkRemoteMetadata() expected agent resolution error, got: %v", err) } } @@ -657,7 +652,7 @@ func TestCheckRemoteMetadata_NoRemoteMetadataBranch(t *testing.T) { // Don't create any remote ref - simulating no remote entire/checkpoints/v1 // Call checkRemoteMetadata - should handle gracefully (no remote branch) - err := checkRemoteMetadata(context.Background(), io.Discard, io.Discard, repo, "nonexistent123") + err := checkRemoteMetadata(context.Background(), id.MustCheckpointID("aaa111bbb222")) if err != nil { t.Errorf("checkRemoteMetadata() returned error when no remote branch: %v", err) } @@ -692,13 +687,13 @@ func TestCheckRemoteMetadata_CheckpointNotOnRemote(t *testing.T) { } // Call checkRemoteMetadata with a DIFFERENT checkpoint ID (not on remote) - err = checkRemoteMetadata(context.Background(), io.Discard, io.Discard, repo, "abcd12345678") + err = checkRemoteMetadata(context.Background(), id.MustCheckpointID("abcd12345678")) if err != nil { t.Errorf("checkRemoteMetadata() returned error for missing checkpoint: %v", err) } } -func TestResumeFromCurrentBranch_FallsBackToRemote(t *testing.T) { +func TestResumeFromCurrentBranch_NoMetadataAvailable(t *testing.T) { tmpDir := t.TempDir() t.Chdir(tmpDir) @@ -712,20 +707,10 @@ func TestResumeFromCurrentBranch_FallsBackToRemote(t *testing.T) { sessionID := "2025-01-01-test-session-uuid" checkpointID := createCheckpointOnMetadataBranch(t, repo, sessionID) - // Copy the local entire/checkpoints/v1 to origin/entire/checkpoints/v1 (simulate remote) - localRef, err := repo.Reference(plumbing.NewBranchReferenceName(paths.MetadataBranchName), true) - if err != nil { - t.Fatalf("Failed to get local metadata branch: %v", err) - } - remoteRef := plumbing.NewHashReference( - plumbing.NewRemoteReferenceName("origin", paths.MetadataBranchName), - localRef.Hash(), - ) - if err := repo.Storer.SetReference(remoteRef); err != nil { - t.Fatalf("Failed to create remote ref: %v", err) - } - - // Delete local entire/checkpoints/v1 branch to simulate "not fetched yet" + // Delete local entire/checkpoints/v1 branch to simulate "not fetched yet". + // Don't create a remote ref — getMetadataTree falls back to + // GetRemoteMetadataBranchTree which reads refs/remotes/origin/... directly, + // so a remote ref would let it succeed without a real fetch. if err := repo.Storer.RemoveReference(plumbing.NewBranchReferenceName(paths.MetadataBranchName)); err != nil { t.Fatalf("Failed to remove local metadata branch: %v", err) } @@ -740,6 +725,7 @@ func TestResumeFromCurrentBranch_FallsBackToRemote(t *testing.T) { } commitMsg := "Add feature\n\nEntire-Checkpoint: " + checkpointID.String() + var err error _, err = w.Commit(commitMsg, &git.CommitOptions{ Author: &object.Signature{ Name: "Test User", @@ -750,17 +736,11 @@ func TestResumeFromCurrentBranch_FallsBackToRemote(t *testing.T) { t.Fatalf("Failed to create commit with checkpoint: %v", err) } - // Run resumeFromCurrentBranch - should fall back to remote and attempt fetch - // In this test environment without a real origin remote, the fetch will fail - // but it should return a SilentError (user-friendly error message already printed) + // Run resumeFromCurrentBranch - no local or remote metadata branch exists, + // so checkRemoteMetadata prints an informational message and returns nil. err = resumeFromCurrentBranch(context.Background(), io.Discard, io.Discard, "master", false) - if err == nil { - t.Error("resumeFromCurrentBranch() should return SilentError when fetch fails") - } else { - var silentErr *SilentError - if !errors.As(err, &silentErr) { - t.Errorf("resumeFromCurrentBranch() should return SilentError, got: %v", err) - } + if err != nil { + t.Errorf("resumeFromCurrentBranch() returned unexpected error: %v", err) } } diff --git a/cmd/entire/cli/strategy/common.go b/cmd/entire/cli/strategy/common.go index 48991c5840..c3d862a21a 100644 --- a/cmd/entire/cli/strategy/common.go +++ b/cmd/entire/cli/strategy/common.go @@ -187,38 +187,31 @@ func ListCheckpoints(ctx context.Context) ([]CheckpointInfo, error) { } // Get details from metadata file (CheckpointSummary format) - if metadataFile, fileErr := checkpointTree.File(paths.MetadataFileName); fileErr == nil { - if content, contentErr := metadataFile.Contents(); contentErr == nil { - var summary checkpoint.CheckpointSummary - if json.Unmarshal([]byte(content), &summary) == nil && len(summary.Sessions) > 0 { - info.CheckpointsCount = summary.CheckpointsCount - info.FilesTouched = summary.FilesTouched - info.SessionCount = len(summary.Sessions) - - // Read session-level metadata for Agent, SessionID, CreatedAt, SessionIDs - for i, sessionPaths := range summary.Sessions { - if sessionPaths.Metadata != "" { - // SessionFilePaths now contains absolute paths with leading "/" - // Strip the leading "/" for tree.File() which expects paths without leading slash - sessionMetadataPath := strings.TrimPrefix(sessionPaths.Metadata, "/") - if sessionFile, sErr := tree.File(sessionMetadataPath); sErr == nil { - if sessionContent, scErr := sessionFile.Contents(); scErr == nil { - var sessionMetadata checkpoint.CommittedMetadata - if json.Unmarshal([]byte(sessionContent), &sessionMetadata) == nil { - info.SessionIDs = append(info.SessionIDs, sessionMetadata.SessionID) - // Use first session's metadata for Agent, SessionID, CreatedAt - if i == 0 { - info.Agent = sessionMetadata.Agent - info.SessionID = sessionMetadata.SessionID - info.CreatedAt = sessionMetadata.CreatedAt - info.IsTask = sessionMetadata.IsTask - info.ToolUseID = sessionMetadata.ToolUseID - } - } - } - } - } - } + if summary, ok := decodeSummaryLiteFromTree(checkpointTree); ok { + info.CheckpointsCount = summary.CheckpointsCount + info.FilesTouched = summary.FilesTouched + info.SessionCount = len(summary.Sessions) + + // Read session-level metadata for Agent, SessionID, CreatedAt, SessionIDs + for i, sessionPaths := range summary.Sessions { + if sessionPaths.Metadata == "" { + continue + } + // SessionFilePaths contains absolute paths with leading "/" + // Strip the leading "/" for tree.File() which expects paths without leading slash + sessionMetadataPath := strings.TrimPrefix(sessionPaths.Metadata, "/") + sessionMeta, sErr := decodeSessionMetadataLite(tree, sessionMetadataPath) + if sErr != nil { + continue + } + info.SessionIDs = append(info.SessionIDs, sessionMeta.SessionID) + // Use first session's metadata for Agent, SessionID, CreatedAt + if i == 0 { + info.Agent = sessionMeta.Agent + info.SessionID = sessionMeta.SessionID + info.CreatedAt = sessionMeta.CreatedAt + info.IsTask = sessionMeta.IsTask + info.ToolUseID = sessionMeta.ToolUseID } } } @@ -436,26 +429,130 @@ func isEmptyMetadataBranch(repo *git.Repository, ref *plumbing.Reference) (bool, return len(tree.Entries) == 0, nil } -// readCheckpointMetadata reads metadata.json from a checkpoint path on entire/checkpoints/v1. +// sessionMetadataLite contains only the fields needed from session-level metadata.json. +// Using a minimal struct avoids allocating large nested objects (Summary, InitialAttribution, +// TokenUsage, etc.) that CommittedMetadata carries but callers never need here. +type sessionMetadataLite struct { + SessionID string `json:"session_id"` + Agent types.AgentType `json:"agent,omitempty"` + CreatedAt time.Time `json:"created_at"` + IsTask bool `json:"is_task,omitempty"` + ToolUseID string `json:"tool_use_id,omitempty"` +} + +// checkpointSummaryLite contains only the fields needed from the root metadata.json. +// Avoids allocating TokenUsage and other heavy fields from CheckpointSummary. +type checkpointSummaryLite struct { + CheckpointID id.CheckpointID `json:"checkpoint_id"` + CheckpointsCount int `json:"checkpoints_count"` + FilesTouched []string `json:"files_touched"` + Sessions []checkpoint.SessionFilePaths `json:"sessions"` +} + +// decodeSessionMetadataLite reads a session metadata.json from the tree using a streaming +// json.Decoder and a minimal struct to avoid allocating large unused fields. +func decodeSessionMetadataLite(tree checkpoint.FileReader, metadataPath string) (*sessionMetadataLite, error) { + file, err := tree.File(metadataPath) + if err != nil { + return nil, fmt.Errorf("session metadata file %s: %w", metadataPath, err) + } + reader, err := file.Reader() + if err != nil { + return nil, fmt.Errorf("session metadata reader %s: %w", metadataPath, err) + } + defer reader.Close() + + var meta sessionMetadataLite + if err := json.NewDecoder(reader).Decode(&meta); err != nil { + return nil, fmt.Errorf("decode session metadata %s: %w", metadataPath, err) + } + return &meta, nil +} + +// decodeSummaryLiteFromTree reads and decodes metadata.json from a checkpoint tree +// using a streaming decoder and minimal struct. Returns the decoded summary and true +// if successful with at least one session, or zero value and false otherwise. +func decodeSummaryLiteFromTree(checkpointTree checkpoint.FileReader) (checkpointSummaryLite, bool) { + metadataFile, fileErr := checkpointTree.File(paths.MetadataFileName) + if fileErr != nil { + return checkpointSummaryLite{}, false + } + reader, readerErr := metadataFile.Reader() + if readerErr != nil { + return checkpointSummaryLite{}, false + } + defer reader.Close() + + var summary checkpointSummaryLite + if err := json.NewDecoder(reader).Decode(&summary); err != nil || len(summary.Sessions) == 0 { + return checkpointSummaryLite{}, false + } + return summary, true +} + +// ReadCheckpointMetadata reads metadata.json from a checkpoint path on entire/checkpoints/v1. // With the new format, root metadata.json is a CheckpointSummary with Agents array. // This function reads the summary and extracts relevant fields into CheckpointInfo, // also reading session-level metadata for IsTask/ToolUseID fields. -func ReadCheckpointMetadata(tree *object.Tree, checkpointPath string) (*CheckpointInfo, error) { +// +// Uses streaming json.Decoder and minimal structs to avoid loading large nested +// objects (Summary, InitialAttribution, TokenUsage) into memory. +func ReadCheckpointMetadata(tree checkpoint.FileReader, checkpointPath string) (*CheckpointInfo, error) { metadataPath := checkpointPath + "/metadata.json" file, err := tree.File(metadataPath) if err != nil { return nil, fmt.Errorf("failed to find metadata at %s: %w", metadataPath, err) } - content, err := file.Contents() + // Session metadata paths in the summary are absolute (e.g., "/ca/b75de47439/0/metadata.json"). + // For a full tree, strip the leading "/" to get tree-relative paths. + normalizePath := func(raw string) string { + return strings.TrimPrefix(raw, "/") + } + return decodeCheckpointInfo(file, tree, checkpointPath, normalizePath) +} + +// ReadCheckpointMetadataFromSubtree reads checkpoint metadata from a tree that is +// already rooted at the checkpoint directory (e.g., after tree.Tree(checkpointID.Path())). +// checkpointPath is the original sharded path (e.g., "ca/b75de47439") and is used +// to strip the prefix from absolute session metadata paths stored in the summary. +func ReadCheckpointMetadataFromSubtree(tree checkpoint.FileReader, checkpointPath string) (*CheckpointInfo, error) { + file, err := tree.File(paths.MetadataFileName) + if err != nil { + return nil, fmt.Errorf("failed to find %s in checkpoint subtree: %w", paths.MetadataFileName, err) + } + + // Session metadata paths are absolute from the tree root (e.g., "/ca/b75de47439/0/metadata.json"). + // Strip the checkpoint prefix to get paths relative to the subtree (e.g., "0/metadata.json"). + prefix := "/" + checkpointPath + "/" + normalizePath := func(raw string) string { + return strings.TrimPrefix(raw, prefix) + } + return decodeCheckpointInfo(file, tree, checkpointPath, normalizePath) +} + +// decodeCheckpointInfo is the shared implementation for ReadCheckpointMetadata and +// ReadCheckpointMetadataFromSubtree. It decodes the root metadata.json, reads +// per-session metadata, and populates a CheckpointInfo. +// +// normalizePath transforms absolute session metadata paths from the summary into +// paths that are valid for tree.File() lookups (the transform differs depending on +// whether tree is a full metadata branch tree or a checkpoint subtree). +func decodeCheckpointInfo( + file checkpoint.FileOpener, + tree checkpoint.FileReader, + checkpointPath string, + normalizePath func(string) string, +) (*CheckpointInfo, error) { + reader, err := file.Reader() if err != nil { return nil, fmt.Errorf("failed to read metadata: %w", err) } + defer reader.Close() - // Try to parse as CheckpointSummary first (new format) - var summary checkpoint.CheckpointSummary - if err := json.Unmarshal([]byte(content), &summary); err == nil { - // If we have sessions array, this is the new format + // Try to parse as CheckpointSummary first (new format) using lite struct + var summary checkpointSummaryLite + if decodeErr := json.NewDecoder(reader).Decode(&summary); decodeErr == nil { if len(summary.Sessions) > 0 { info := &CheckpointInfo{ CheckpointID: summary.CheckpointID, @@ -467,40 +564,46 @@ func ReadCheckpointMetadata(tree *object.Tree, checkpointPath string) (*Checkpoi // Read all sessions' metadata to populate SessionIDs and get other fields from first session var sessionIDs []string for i, sessionPaths := range summary.Sessions { - if sessionPaths.Metadata != "" { - // SessionFilePaths now contains absolute paths with leading "/" - // Strip the leading "/" for tree.File() which expects paths without leading slash - sessionMetadataPath := strings.TrimPrefix(sessionPaths.Metadata, "/") - if sessionFile, err := tree.File(sessionMetadataPath); err == nil { - if sessionContent, err := sessionFile.Contents(); err == nil { - var sessionMetadata checkpoint.CommittedMetadata - if json.Unmarshal([]byte(sessionContent), &sessionMetadata) == nil { - sessionIDs = append(sessionIDs, sessionMetadata.SessionID) - // Use first session for Agent, SessionID, CreatedAt, IsTask, ToolUseID - if i == 0 { - info.Agent = sessionMetadata.Agent - info.SessionID = sessionMetadata.SessionID - info.CreatedAt = sessionMetadata.CreatedAt - info.IsTask = sessionMetadata.IsTask - info.ToolUseID = sessionMetadata.ToolUseID - } - } - } - } + if sessionPaths.Metadata == "" { + continue + } + sessionMetadataPath := normalizePath(sessionPaths.Metadata) + sessionMeta, sErr := decodeSessionMetadataLite(tree, sessionMetadataPath) + if sErr != nil { + logging.Debug(context.Background(), "decodeCheckpointInfo: session metadata decode failed", + slog.Int("session_index", i), + slog.String("metadata_path", sessionMetadataPath), + slog.String("checkpoint_path", checkpointPath), + slog.String("error", sErr.Error()), + ) + continue + } + sessionIDs = append(sessionIDs, sessionMeta.SessionID) + if i == 0 { + info.Agent = sessionMeta.Agent + info.SessionID = sessionMeta.SessionID + info.CreatedAt = sessionMeta.CreatedAt + info.IsTask = sessionMeta.IsTask + info.ToolUseID = sessionMeta.ToolUseID } } info.SessionIDs = sessionIDs - return info, nil } } - // Fall back to parsing as CheckpointInfo (old format or direct info) + // Fall back to parsing as CheckpointInfo (old format or direct info). + // Re-read the file since the decoder consumed the reader. + fallbackReader, err := file.Reader() + if err != nil { + return nil, fmt.Errorf("failed to re-read metadata: %w", err) + } + defer fallbackReader.Close() + var metadata CheckpointInfo - if err := json.Unmarshal([]byte(content), &metadata); err != nil { + if err := json.NewDecoder(fallbackReader).Decode(&metadata); err != nil { return nil, fmt.Errorf("failed to parse metadata: %w", err) } - return &metadata, nil } diff --git a/cmd/entire/cli/strategy/common_helpers_test.go b/cmd/entire/cli/strategy/common_helpers_test.go new file mode 100644 index 0000000000..30fcc7890f --- /dev/null +++ b/cmd/entire/cli/strategy/common_helpers_test.go @@ -0,0 +1,84 @@ +package strategy + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + + "github.com/go-git/go-git/v6/plumbing/object" +) + +// readCheckpointMetadataFull is the original ReadCheckpointMetadata implementation +// preserved for test code that needs the full deserialization behavior (loading +// every field from both the root CheckpointSummary and per-session CommittedMetadata). +// +// Production code uses ReadCheckpointMetadata which streams via json.Decoder +// and uses minimal structs to avoid allocating large unused fields +// (Summary, InitialAttribution, TokenUsage, etc.). +// +//nolint:unused // Test helper preserved for tests that need full deserialization +func readCheckpointMetadataFull(tree *object.Tree, checkpointPath string) (*CheckpointInfo, error) { + metadataPath := checkpointPath + "/metadata.json" + file, err := tree.File(metadataPath) + if err != nil { + return nil, fmt.Errorf("failed to find metadata at %s: %w", metadataPath, err) + } + + content, err := file.Contents() + if err != nil { + return nil, fmt.Errorf("failed to read metadata: %w", err) + } + + // Try to parse as CheckpointSummary first (new format) + var summary checkpoint.CheckpointSummary + if err := json.Unmarshal([]byte(content), &summary); err == nil { + // If we have sessions array, this is the new format + if len(summary.Sessions) > 0 { + info := &CheckpointInfo{ + CheckpointID: summary.CheckpointID, + CheckpointsCount: summary.CheckpointsCount, + FilesTouched: summary.FilesTouched, + SessionCount: len(summary.Sessions), + } + + // Read all sessions' metadata to populate SessionIDs and get other fields from first session + var sessionIDs []string + for i, sessionPaths := range summary.Sessions { + if sessionPaths.Metadata != "" { + // SessionFilePaths now contains absolute paths with leading "/" + // Strip the leading "/" for tree.File() which expects paths without leading slash + sessionMetadataPath := strings.TrimPrefix(sessionPaths.Metadata, "/") + if sessionFile, err := tree.File(sessionMetadataPath); err == nil { + if sessionContent, err := sessionFile.Contents(); err == nil { + var sessionMetadata checkpoint.CommittedMetadata + if json.Unmarshal([]byte(sessionContent), &sessionMetadata) == nil { + sessionIDs = append(sessionIDs, sessionMetadata.SessionID) + // Use first session for Agent, SessionID, CreatedAt, IsTask, ToolUseID + if i == 0 { + info.Agent = sessionMetadata.Agent + info.SessionID = sessionMetadata.SessionID + info.CreatedAt = sessionMetadata.CreatedAt + info.IsTask = sessionMetadata.IsTask + info.ToolUseID = sessionMetadata.ToolUseID + } + } + } + } + } + } + info.SessionIDs = sessionIDs + + return info, nil + } + } + + // Fall back to parsing as CheckpointInfo (old format or direct info) + var metadata CheckpointInfo + if err := json.Unmarshal([]byte(content), &metadata); err != nil { + return nil, fmt.Errorf("failed to parse metadata: %w", err) + } + + return &metadata, nil +} diff --git a/cmd/entire/cli/strategy/hooks_test.go b/cmd/entire/cli/strategy/hooks_test.go index 4fa7c0e0c2..ea8e68b2b5 100644 --- a/cmd/entire/cli/strategy/hooks_test.go +++ b/cmd/entire/cli/strategy/hooks_test.go @@ -11,6 +11,18 @@ import ( "github.com/entireio/cli/cmd/entire/cli/paths" ) +// clearGlobalHooksPath overrides any global core.hooksPath setting so that +// test repos use their default .git/hooks directory. Setting the local value +// takes precedence over the global one. +func clearGlobalHooksPath(t *testing.T, repoDir string) { + t.Helper() + cmd := exec.CommandContext(context.Background(), "git", "config", "--local", "core.hooksPath", filepath.Join(repoDir, ".git", "hooks")) + cmd.Dir = repoDir + if err := cmd.Run(); err != nil { + t.Fatalf("failed to set local core.hooksPath: %v", err) + } +} + // initHooksTestRepo creates a temporary git repository, changes to it, and clears // the repo root cache. Returns the repo directory path and the hooks directory path. func initHooksTestRepo(t *testing.T) (string, string) { @@ -24,6 +36,7 @@ func initHooksTestRepo(t *testing.T) (string, string) { if err := cmd.Run(); err != nil { t.Fatalf("failed to init git repo: %v", err) } + clearGlobalHooksPath(t, tmpDir) paths.ClearWorktreeRootCache() return tmpDir, filepath.Join(tmpDir, ".git", "hooks") @@ -82,6 +95,7 @@ func TestGetGitDirInPath_Worktree(t *testing.T) { if err := cmd.Run(); err != nil { t.Fatalf("failed to init main repo: %v", err) } + clearGlobalHooksPath(t, mainRepo) // Configure git user for the commit cmd = exec.CommandContext(ctx, "git", "config", "user.email", "test@test.com") @@ -177,6 +191,7 @@ func TestGetHooksDirInPath_RegularRepo(t *testing.T) { if err := cmd.Run(); err != nil { t.Fatalf("failed to init git repo: %v", err) } + clearGlobalHooksPath(t, tmpDir) result, err := getHooksDirInPath(context.Background(), tmpDir) if err != nil { @@ -334,6 +349,7 @@ func initHooksWorktreeRepo(t *testing.T) (string, string) { if err := cmd.Run(); err != nil { t.Fatalf("failed to init main repo: %v", err) } + clearGlobalHooksPath(t, mainRepo) cmd = exec.CommandContext(ctx, "git", "config", "user.email", "test@test.com") cmd.Dir = mainRepo diff --git a/cmd/entire/cli/strategy/manual_commit.go b/cmd/entire/cli/strategy/manual_commit.go index 805dc26be3..5e36d1b493 100644 --- a/cmd/entire/cli/strategy/manual_commit.go +++ b/cmd/entire/cli/strategy/manual_commit.go @@ -26,6 +26,10 @@ type ManualCommitStrategy struct { checkpointStoreOnce sync.Once // checkpointStoreErr captures any error during initialization checkpointStoreErr error + + // blobFetcher, when set, is passed to the checkpoint store to enable + // on-demand blob fetching after treeless fetches. Set via SetBlobFetcher. + blobFetcher checkpoint.BlobFetchFunc } // getStateStore returns the session state store, initializing it lazily if needed. @@ -52,7 +56,11 @@ func (s *ManualCommitStrategy) getCheckpointStore() (*checkpoint.GitStore, error return } WarnIfMetadataDisconnected() - s.checkpointStore = checkpoint.NewGitStore(repo) + store := checkpoint.NewGitStore(repo) + if s.blobFetcher != nil { + store.SetBlobFetcher(s.blobFetcher) + } + s.checkpointStore = store }) return s.checkpointStore, s.checkpointStoreErr } @@ -62,6 +70,18 @@ func NewManualCommitStrategy() *ManualCommitStrategy { return &ManualCommitStrategy{} } +// SetBlobFetcher configures on-demand blob fetching for the checkpoint store. +// Must be called before the first checkpoint store access (e.g., before RestoreLogsOnly). +func (s *ManualCommitStrategy) SetBlobFetcher(f checkpoint.BlobFetchFunc) { + s.blobFetcher = f +} + +// HasBlobFetcher reports whether a blob fetcher is configured. +// Used in tests to verify the strategy is properly wired for treeless fetch support. +func (s *ManualCommitStrategy) HasBlobFetcher() bool { + return s.blobFetcher != nil +} + // ValidateRepository validates that the repository is suitable for this strategy. func (s *ManualCommitStrategy) ValidateRepository() error { repo, err := OpenRepository(context.Background()) diff --git a/cmd/entire/cli/strategy/readcheckpoint_bench_test.go b/cmd/entire/cli/strategy/readcheckpoint_bench_test.go new file mode 100644 index 0000000000..7fddb01725 --- /dev/null +++ b/cmd/entire/cli/strategy/readcheckpoint_bench_test.go @@ -0,0 +1,233 @@ +package strategy + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/entireio/cli/cmd/entire/cli/benchutil" + "github.com/entireio/cli/cmd/entire/cli/paths" + + gogit "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" +) + +// BenchmarkReadCheckpointMetadata measures the time to read and decode a single +// checkpoint's metadata from the entire/checkpoints/v1 branch. +// Tests both the streaming lite decoder and varying numbers of sessions. +func BenchmarkReadCheckpointMetadata(b *testing.B) { + b.Run("1Checkpoint", benchReadCheckpointMetadata(1)) + b.Run("10Checkpoints", benchReadCheckpointMetadata(10)) + b.Run("50Checkpoints", benchReadCheckpointMetadata(50)) + b.Run("200Checkpoints", benchReadCheckpointMetadata(200)) +} + +func benchReadCheckpointMetadata(checkpointCount int) func(*testing.B) { + return func(b *testing.B) { + repo := benchutil.NewBenchRepo(b, benchutil.RepoOpts{FileCount: 10}) + repo.SeedMetadataBranch(b, checkpointCount) + + // Get the metadata branch tree and find a checkpoint path to read + tree, cpPath := getMetadataBranchCheckpoint(b, repo.Repo) + + b.ResetTimer() + b.ReportMetric(float64(checkpointCount), "total_checkpoints") + + for b.Loop() { + info, err := ReadCheckpointMetadata(tree, cpPath) + if err != nil { + b.Fatalf("ReadCheckpointMetadata: %v", err) + } + if info.SessionID == "" { + b.Fatal("expected non-empty SessionID") + } + } + } +} + +// BenchmarkListCheckpoints measures the full ListCheckpoints path which iterates +// all sharded checkpoints and decodes each one using the lite streaming decoder. +func BenchmarkListCheckpoints(b *testing.B) { + b.Run("1Checkpoint", benchListCheckpoints(1)) + b.Run("10Checkpoints", benchListCheckpoints(10)) + b.Run("50Checkpoints", benchListCheckpoints(50)) + b.Run("200Checkpoints", benchListCheckpoints(200)) +} + +func benchListCheckpoints(checkpointCount int) func(*testing.B) { + return func(b *testing.B) { + repo := benchutil.NewBenchRepo(b, benchutil.RepoOpts{FileCount: 10}) + repo.SeedMetadataBranch(b, checkpointCount) + + b.Chdir(repo.Dir) + paths.ClearWorktreeRootCache() + + b.ResetTimer() + b.ReportMetric(float64(checkpointCount), "total_checkpoints") + + for b.Loop() { + checkpoints, err := ListCheckpoints(context.Background()) + if err != nil { + b.Fatalf("ListCheckpoints: %v", err) + } + if len(checkpoints) != checkpointCount { + b.Fatalf("expected %d checkpoints, got %d", checkpointCount, len(checkpoints)) + } + } + } +} + +// BenchmarkDecodeSummaryLiteFromTree measures just the root metadata.json +// decoding using the lite streaming decoder. +func BenchmarkDecodeSummaryLiteFromTree(b *testing.B) { + repo := benchutil.NewBenchRepo(b, benchutil.RepoOpts{FileCount: 10}) + repo.SeedMetadataBranch(b, 5) + + cpTree := getCheckpointTree(b, repo.Repo) + + b.ResetTimer() + for b.Loop() { + summary, ok := decodeSummaryLiteFromTree(cpTree) + if !ok { + b.Fatal("decodeSummaryLiteFromTree returned false") + } + if summary.CheckpointID == "" { + b.Fatal("expected non-empty CheckpointID") + } + } +} + +// BenchmarkDecodeSessionMetadataLite measures decoding a single session +// metadata.json using the lite streaming decoder. +func BenchmarkDecodeSessionMetadataLite(b *testing.B) { + repo := benchutil.NewBenchRepo(b, benchutil.RepoOpts{FileCount: 10}) + repo.SeedMetadataBranch(b, 5) + + // Get tree and find a session metadata path + tree := getMetadataBranchTree(b, repo.Repo) + sessionPath := findSessionMetadataPath(b, repo.Repo, tree) + + b.ResetTimer() + for b.Loop() { + meta, err := decodeSessionMetadataLite(tree, sessionPath) + if err != nil { + b.Fatalf("decodeSessionMetadataLite: %v", err) + } + if meta.SessionID == "" { + b.Fatal("expected non-empty SessionID") + } + } +} + +// --- helpers --- + +// getMetadataBranchTree returns the root tree of entire/checkpoints/v1. +func getMetadataBranchTree(b *testing.B, repo *gogit.Repository) *object.Tree { + b.Helper() + ref, err := repo.Reference(plumbing.NewBranchReferenceName(paths.MetadataBranchName), true) + if err != nil { + b.Fatalf("get metadata branch ref: %v", err) + } + commit, err := repo.CommitObject(ref.Hash()) + if err != nil { + b.Fatalf("get metadata branch commit: %v", err) + } + tree, err := commit.Tree() + if err != nil { + b.Fatalf("get metadata branch tree: %v", err) + } + return tree +} + +// getMetadataBranchCheckpoint returns the metadata branch tree and a checkpoint path +// (e.g., "ab/cdef123456") for benchmarking ReadCheckpointMetadata. +func getMetadataBranchCheckpoint(b *testing.B, repo *gogit.Repository) (*object.Tree, string) { + b.Helper() + tree := getMetadataBranchTree(b, repo) + + // Find first valid checkpoint path: <2-char-bucket>/ + for _, bucketEntry := range tree.Entries { + if bucketEntry.Mode != filemode.Dir || len(bucketEntry.Name) != 2 { + continue + } + bucketTree, err := repo.TreeObject(bucketEntry.Hash) + if err != nil { + continue + } + for _, cpEntry := range bucketTree.Entries { + if cpEntry.Mode != filemode.Dir { + continue + } + return tree, fmt.Sprintf("%s/%s", bucketEntry.Name, cpEntry.Name) + } + } + b.Fatal("no checkpoint found on metadata branch") + return nil, "" +} + +// getCheckpointTree returns the tree object for a single checkpoint directory. +func getCheckpointTree(b *testing.B, repo *gogit.Repository) *object.Tree { + b.Helper() + tree := getMetadataBranchTree(b, repo) + + for _, bucketEntry := range tree.Entries { + if bucketEntry.Mode != filemode.Dir || len(bucketEntry.Name) != 2 { + continue + } + bucketTree, err := repo.TreeObject(bucketEntry.Hash) + if err != nil { + continue + } + for _, cpEntry := range bucketTree.Entries { + if cpEntry.Mode != filemode.Dir { + continue + } + cpTree, err := repo.TreeObject(cpEntry.Hash) + if err != nil { + continue + } + return cpTree + } + } + b.Fatal("no checkpoint tree found") + return nil +} + +// findSessionMetadataPath finds the path to a session metadata.json on the +// metadata branch tree (stripping leading "/" from SessionFilePaths). +func findSessionMetadataPath(b *testing.B, repo *gogit.Repository, tree *object.Tree) string { + b.Helper() + + for _, bucketEntry := range tree.Entries { + if bucketEntry.Mode != filemode.Dir || len(bucketEntry.Name) != 2 { + continue + } + bucketTree, err := repo.TreeObject(bucketEntry.Hash) + if err != nil { + continue + } + for _, cpEntry := range bucketTree.Entries { + if cpEntry.Mode != filemode.Dir { + continue + } + cpTree, err := repo.TreeObject(cpEntry.Hash) + if err != nil { + continue + } + // Look for metadata.json in the checkpoint tree + summary, ok := decodeSummaryLiteFromTree(cpTree) + if !ok || len(summary.Sessions) == 0 { + continue + } + path := strings.TrimPrefix(summary.Sessions[0].Metadata, "/") + if path != "" { + return path + } + } + } + b.Fatal("no session metadata path found") + return "" +} diff --git a/e2e/tests/resume_remote_test.go b/e2e/tests/resume_remote_test.go new file mode 100644 index 0000000000..a587fab24f --- /dev/null +++ b/e2e/tests/resume_remote_test.go @@ -0,0 +1,124 @@ +//go:build e2e + +package tests + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/entireio/cli/e2e/entire" + "github.com/entireio/cli/e2e/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestResumeFromClonedRepo: agent creates a file on a feature branch and user +// commits, then the repo is cloned (simulating a teammate). The clone has no +// local entire/checkpoints/v1 branch. `entire resume feature` should fetch +// the metadata branch automatically and restore the session. +func TestResumeFromClonedRepo(t *testing.T) { + testutil.ForEachAgent(t, 3*time.Minute, func(t *testing.T, s *testutil.RepoState, ctx context.Context) { + // Set up a bare remote so we can push and clone. + bareDir := testutil.SetupBareRemote(t, s) + + // Commit files from `entire enable` so main has a clean working tree. + s.Git(t, "add", ".") + s.Git(t, "commit", "-m", "Enable entire") + s.Git(t, "push") + + // Do agent work on a feature branch. + s.Git(t, "checkout", "-b", "feature") + + _, err := s.RunPrompt(t, ctx, + "create a file at docs/hello.md with a paragraph about greetings. Do not ask for confirmation, just make the change.") + if err != nil { + t.Fatalf("agent failed: %v", err) + } + + s.Git(t, "add", ".") + s.Git(t, "commit", "-m", "Add hello doc") + testutil.WaitForCheckpoint(t, s, 15*time.Second) + + // Push feature branch and metadata branch to the bare remote. + s.Git(t, "push", "-u", "origin", "feature") + s.Git(t, "push", "origin", "entire/checkpoints/v1:entire/checkpoints/v1") + + // Clone the repo to a new directory (simulating a teammate). + cloneDir := t.TempDir() + if resolved, symErr := filepath.EvalSymlinks(cloneDir); symErr == nil { + cloneDir = resolved + } + // Remove the dir because git clone wants to create it + require.NoError(t, os.RemoveAll(cloneDir)) + testutil.Git(t, "", "clone", bareDir, cloneDir) + testutil.Git(t, cloneDir, "config", "user.name", "E2E Clone") + testutil.Git(t, cloneDir, "config", "user.email", "e2e-clone@test.local") + + // Verify the metadata branch does NOT exist locally in the clone. + _, err = testutil.GitOutputErr(cloneDir, "rev-parse", "--verify", "refs/heads/entire/checkpoints/v1") + require.Error(t, err, "metadata branch should not exist locally in clone") + + // Create the local feature branch (clone only has origin/feature as remote tracking ref). + // Then switch back to the default branch so resume can switch to it. + mainClone := testutil.GitOutput(t, cloneDir, "branch", "--show-current") + testutil.Git(t, cloneDir, "checkout", "feature") + testutil.Git(t, cloneDir, "checkout", mainClone) + + // Enable entire in the cloned repo and commit the enable files. + entire.Enable(t, cloneDir, s.Agent.EntireAgent()) + testutil.Git(t, cloneDir, "add", ".") + testutil.Git(t, cloneDir, "commit", "-m", "Enable entire in clone") + + // Run resume from the clone — should fetch metadata and succeed. + out, err := entire.Resume(cloneDir, "feature") + require.NoError(t, err, "entire resume failed in clone: %s", out) + + current := testutil.GitOutput(t, cloneDir, "branch", "--show-current") + assert.Equal(t, "feature", current, "should be on feature branch after resume") + assert.Contains(t, out, "To continue", "resume output should show resume instructions") + + // Verify the metadata branch now exists locally. + _, err = testutil.GitOutputErr(cloneDir, "rev-parse", "--verify", "refs/heads/entire/checkpoints/v1") + assert.NoError(t, err, "metadata branch should exist locally after resume") + }) +} + +// TestResumeMetadataBranchAlreadyLocal: same setup as TestResumeFromClonedRepo +// but the metadata branch already exists locally. Resume should still work +// (fetch updates local to latest). +func TestResumeMetadataBranchAlreadyLocal(t *testing.T) { + testutil.ForEachAgent(t, 3*time.Minute, func(t *testing.T, s *testutil.RepoState, ctx context.Context) { + mainBranch := testutil.GitOutput(t, s.Dir, "branch", "--show-current") + + // Commit files from `entire enable` so main has a clean working tree. + s.Git(t, "add", ".") + s.Git(t, "commit", "-m", "Enable entire") + + // Do agent work on a feature branch. + s.Git(t, "checkout", "-b", "feature") + + _, err := s.RunPrompt(t, ctx, + "create a file at docs/hello.md with a paragraph about greetings. Do not ask for confirmation, just make the change.") + if err != nil { + t.Fatalf("agent failed: %v", err) + } + + s.Git(t, "add", ".") + s.Git(t, "commit", "-m", "Add hello doc") + testutil.WaitForCheckpoint(t, s, 15*time.Second) + + // Switch back to main and resume the feature branch. + // The metadata branch exists locally (was created during commit). + s.Git(t, "checkout", mainBranch) + + out, err := entire.Resume(s.Dir, "feature") + require.NoError(t, err, "entire resume failed: %s", out) + + current := testutil.GitOutput(t, s.Dir, "branch", "--show-current") + assert.Equal(t, "feature", current, "should be on feature branch after resume") + assert.Contains(t, out, "To continue", "resume output should show resume instructions") + }) +}