From 259ec89970a8390b1a5382040f388d214a8d3964 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 24 Apr 2026 13:58:39 +0100 Subject: [PATCH 01/12] Gate tree-only fetches on filtered_fetches Entire-Checkpoint: 8436c49de6b9 --- cmd/entire/cli/resume.go | 43 ++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/cmd/entire/cli/resume.go b/cmd/entire/cli/resume.go index 129cf9af52..545d21bab9 100644 --- a/cmd/entire/cli/resume.go +++ b/cmd/entire/cli/resume.go @@ -456,26 +456,30 @@ func getMetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) ) } - // Try treeless fetch from origin - if fetchErr := FetchMetadataTreeOnly(ctx); fetchErr == nil { - freshRepo, repoErr := openRepository(ctx) - if repoErr == nil { - logRefHash(freshRepo, "treeless-fetch") - metadataTree, treeErr := strategy.GetMetadataBranchTree(freshRepo) - if treeErr == nil { - logging.Debug(logCtx, "metadata tree obtained via treeless fetch", - slog.String("tree_hash", metadataTree.Hash.String()), + // Only use the tree-only fetch path when filtered fetches are enabled. + // Otherwise this would degrade into an ordinary shallow fetch while the + // surrounding control flow still assumes the tree-only fast path. + if settings.IsFilteredFetchesEnabled(ctx) { + if fetchErr := FetchMetadataTreeOnly(ctx); fetchErr == nil { + freshRepo, repoErr := openRepository(ctx) + if repoErr == nil { + logRefHash(freshRepo, "treeless-fetch") + metadataTree, treeErr := strategy.GetMetadataBranchTree(freshRepo) + if treeErr == nil { + logging.Debug(logCtx, "metadata tree obtained via treeless fetch", + slog.String("tree_hash", metadataTree.Hash.String()), + ) + return metadataTree, freshRepo, nil + } + logging.Debug(logCtx, "treeless fetch succeeded but tree read failed", + slog.String("error", treeErr.Error()), ) - return metadataTree, freshRepo, nil } - logging.Debug(logCtx, "treeless fetch succeeded but tree read failed", - slog.String("error", treeErr.Error()), + } else { + logging.Debug(logCtx, "treeless fetch failed, trying local", + slog.String("error", fetchErr.Error()), ) } - } else { - logging.Debug(logCtx, "treeless fetch failed, trying local", - slog.String("error", fetchErr.Error()), - ) } // Try local (may have been set by a prior fetch or push) @@ -537,7 +541,12 @@ func getMetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) // getV2MetadataTree resolves the v2 /main ref tree with the same // fetch fallback pattern as getMetadataTree, including checkpoint remote support. func getV2MetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) { - tree, repo, err := checkpoint.GetV2MetadataTree(ctx, FetchV2MainTreeOnly, FetchV2MainRef, openRepository) + var treeOnlyFetch checkpoint.FetchRefFunc + if settings.IsFilteredFetchesEnabled(ctx) { + treeOnlyFetch = FetchV2MainTreeOnly + } + + tree, repo, err := checkpoint.GetV2MetadataTree(ctx, treeOnlyFetch, FetchV2MainRef, openRepository) if err == nil { return tree, repo, nil } From 5c319e6a01760c7bae81d370b5fb231f29c22337 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 24 Apr 2026 14:20:50 +0100 Subject: [PATCH 02/12] Remove shallow checkpoint fetches Entire-Checkpoint: 2aaa0ca9a3ae --- cmd/entire/cli/checkpoint/remote/git.go | 4 ---- cmd/entire/cli/git_operations.go | 32 ++++++++++--------------- cmd/entire/cli/resume.go | 2 +- cmd/entire/cli/strategy/common.go | 15 +++++------- 4 files changed, 20 insertions(+), 33 deletions(-) diff --git a/cmd/entire/cli/checkpoint/remote/git.go b/cmd/entire/cli/checkpoint/remote/git.go index e24cdc938d..33e1995264 100644 --- a/cmd/entire/cli/checkpoint/remote/git.go +++ b/cmd/entire/cli/checkpoint/remote/git.go @@ -31,7 +31,6 @@ var sshTokenWarningOnce sync.Once //nolint:gochecknoglobals // intentional per-p type FetchOptions struct { Remote string // remote name or URL (required) RefSpecs []string // one or more refspecs / object hashes - Shallow bool // adds --depth=1 NoTags bool // adds --no-tags NoFilter bool // when true, skips --filter=blob:none even if filtered fetches are enabled Dir string // working directory (empty = CWD) @@ -50,9 +49,6 @@ func Fetch(ctx context.Context, opts FetchOptions) ([]byte, error) { if opts.NoTags { args = append(args, "--no-tags") } - if opts.Shallow { - args = append(args, "--depth=1") - } args = append(args, opts.ExtraArgs...) if !opts.NoFilter && settings.IsFilteredFetchesEnabled(ctx) { args = append(args, "--filter=blob:none") diff --git a/cmd/entire/cli/git_operations.go b/cmd/entire/cli/git_operations.go index 16076d78b3..4723fa20c4 100644 --- a/cmd/entire/cli/git_operations.go +++ b/cmd/entire/cli/git_operations.go @@ -401,23 +401,21 @@ func FetchAndCheckoutRemoteBranch(ctx context.Context, branchName string) error // creates/updates the local branch. The fetch is unfiltered (no --filter=blob:none) // because callers (resume, explain) need blob content, not just tree structure. func FetchMetadataBranch(ctx context.Context) error { - return fetchMetadataFromOrigin(ctx, false /* shallow */, true /* noFilter */) + return fetchMetadataFromOrigin(ctx, true /* noFilter */) } -// FetchMetadataTreeOnly fetches the tip of the entire/checkpoints/v1 branch -// from origin with --depth=1, downloading only the latest commit and its tree -// objects. After this call, tree navigation via go-git works but blob reads -// will fail for objects that weren't previously fetched. +// FetchMetadataTreeOnly fetches the entire/checkpoints/v1 branch from origin +// without blobs. After this call, tree navigation via go-git works but blob +// reads will fail for objects that weren't previously fetched. func FetchMetadataTreeOnly(ctx context.Context) error { - return fetchMetadataFromOrigin(ctx, true /* shallow */, false /* noFilter */) + return fetchMetadataFromOrigin(ctx, false /* noFilter */) } // fetchMetadataFromOrigin fetches the v1 metadata branch from origin into the // remote-tracking ref refs/remotes/origin/, then safely advances the -// local branch to match. When shallow is true, --depth=1 is added so only -// the tip is downloaded. When noFilter is true, --filter=blob:none is suppressed +// local branch to match. When noFilter is true, --filter=blob:none is suppressed // so blob content is included. -func fetchMetadataFromOrigin(ctx context.Context, shallow, noFilter bool) error { +func fetchMetadataFromOrigin(ctx context.Context, noFilter bool) error { branchName := paths.MetadataBranchName ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) @@ -434,7 +432,6 @@ func fetchMetadataFromOrigin(ctx context.Context, shallow, noFilter bool) error Remote: fetchTarget, RefSpecs: []string{refSpec}, NoTags: true, - Shallow: shallow, NoFilter: noFilter, }) if fetchErr != nil { @@ -459,25 +456,23 @@ func fetchMetadataFromOrigin(ctx context.Context, shallow, noFilter bool) error return nil } -// FetchV2MainTreeOnly fetches the tip of the v2 /main ref from origin with -// --depth=1, downloading only the latest commit and its tree objects. +// FetchV2MainTreeOnly fetches the v2 /main ref from origin without blobs. // Uses explicit refspec since v2 refs are under refs/entire/, not refs/heads/. func FetchV2MainTreeOnly(ctx context.Context) error { - return fetchV2MainFromOrigin(ctx, true /* shallow */, false /* noFilter */) + return fetchV2MainFromOrigin(ctx, false /* noFilter */) } // FetchV2MainRef fetches the v2 /main ref from origin with full blob content. // The fetch is unfiltered so resume/explain can read metadata JSON blobs. // Uses explicit refspec since v2 refs are under refs/entire/, not refs/heads/. func FetchV2MainRef(ctx context.Context) error { - return fetchV2MainFromOrigin(ctx, false /* shallow */, true /* noFilter */) + return fetchV2MainFromOrigin(ctx, true /* noFilter */) } // fetchV2MainFromOrigin fetches the v2 /main ref from origin into the shared -// staging ref, then promotes it via strategy.PromoteTmpRefSafely. When -// shallow is true, --depth=1 is added so only the tip is downloaded. -// When noFilter is true, --filter=blob:none is suppressed. -func fetchV2MainFromOrigin(ctx context.Context, shallow, noFilter bool) error { +// staging ref, then promotes it via strategy.PromoteTmpRefSafely. When noFilter +// is true, --filter=blob:none is suppressed. +func fetchV2MainFromOrigin(ctx context.Context, noFilter bool) error { ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() @@ -492,7 +487,6 @@ func fetchV2MainFromOrigin(ctx context.Context, shallow, noFilter bool) error { Remote: fetchTarget, RefSpecs: []string{refSpec}, NoTags: true, - Shallow: shallow, NoFilter: noFilter, }) if fetchErr != nil { diff --git a/cmd/entire/cli/resume.go b/cmd/entire/cli/resume.go index 545d21bab9..695c79136c 100644 --- a/cmd/entire/cli/resume.go +++ b/cmd/entire/cli/resume.go @@ -457,7 +457,7 @@ func getMetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) } // Only use the tree-only fetch path when filtered fetches are enabled. - // Otherwise this would degrade into an ordinary shallow fetch while the + // Otherwise this would degrade into an ordinary full fetch while the // surrounding control flow still assumes the tree-only fast path. if settings.IsFilteredFetchesEnabled(ctx) { if fetchErr := FetchMetadataTreeOnly(ctx); fetchErr == nil { diff --git a/cmd/entire/cli/strategy/common.go b/cmd/entire/cli/strategy/common.go index 104a138427..788a143c35 100644 --- a/cmd/entire/cli/strategy/common.go +++ b/cmd/entire/cli/strategy/common.go @@ -129,22 +129,19 @@ func PromoteTmpRefSafely(ctx context.Context, tmpRefName, destRefName plumbing.R // SafelyAdvanceLocalRef updates localRefName to point at targetHash only when // the move is non-destructive: // -// - local missing → create at target -// - local == target → no-op -// - local at or ahead of target (target is an ancestor of local) → no-op -// - local strictly behind target (local is an ancestor of target) → fast-forward +// - local missing: create at target +// - local == target: no-op +// - local at or ahead of target (target is an ancestor of local): no-op +// - local strictly behind target (local is an ancestor of target): fast-forward // - diverged or unrelated history (neither ref is an ancestor of the other) -// → no-op, logged at debug level +// no-op, logged at debug level // // The diverged case is the dangerous one this guard exists for. The CLI // maintains orphan-style refs (entire/checkpoints/v1, the V2 main ref) where -// unpushed local commits encode user work — a fetch that landed sibling +// unpushed local commits encode user work. A fetch that landed sibling // commits from another machine must not silently rewind that work. In the // diverged case readers fall through to the remote-tracking tree or to v1, // so resume keeps working while local-only commits stay reachable. -// -// The ancestry checks walk from the local ref (which has full history), so -// callers that fetched with --depth=1 do not break the check. func SafelyAdvanceLocalRef(ctx context.Context, repo *git.Repository, localRefName plumbing.ReferenceName, targetHash plumbing.Hash) error { currentLocal, localErr := repo.Reference(localRefName, true) if localErr == nil { From 9a66f8bccf934f7ba365d4bdeba4059de303fd94 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 24 Apr 2026 14:57:49 +0100 Subject: [PATCH 03/12] Unshallow filtered checkpoint fetches Entire-Checkpoint: 5dec9ad3f03b --- cmd/entire/cli/checkpoint/remote/git.go | 47 +++++++++++++- cmd/entire/cli/checkpoint/remote/git_test.go | 64 ++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/cmd/entire/cli/checkpoint/remote/git.go b/cmd/entire/cli/checkpoint/remote/git.go index 33e1995264..95f1f0f5cb 100644 --- a/cmd/entire/cli/checkpoint/remote/git.go +++ b/cmd/entire/cli/checkpoint/remote/git.go @@ -39,6 +39,9 @@ type FetchOptions struct { // Fetch runs git fetch with checkpoint token injection and optional // filtered fetches (--filter=blob:none when settings enable it). +// When filtered_fetches is enabled and the repository is shallow, Fetch also +// adds --unshallow. This migrates users away from legacy shallow checkpoint +// repositories because some Entire commands need checkpoint history. // GIT_TERMINAL_PROMPT=0 is always set. // // Callers that pass a remote name (e.g., "origin") and want filtered fetches to @@ -50,7 +53,14 @@ func Fetch(ctx context.Context, opts FetchOptions) ([]byte, error) { args = append(args, "--no-tags") } args = append(args, opts.ExtraArgs...) - if !opts.NoFilter && settings.IsFilteredFetchesEnabled(ctx) { + filteredFetchesEnabled := settings.IsFilteredFetchesEnabled(ctx) + if filteredFetchesEnabled && isShallowRepository(ctx, opts.Dir) { + // Filtered checkpoint fetches used to create shallow repositories. + // Unshallow on subsequent filtered fetches so commands that rely on + // checkpoint ancestry/history can operate correctly. + args = append(args, "--unshallow") + } + if !opts.NoFilter && filteredFetchesEnabled { args = append(args, "--filter=blob:none") } args = append(args, opts.Remote) @@ -305,6 +315,41 @@ func ResolveFetchTarget(ctx context.Context, target string) (string, error) { return url, nil } +func isShallowRepository(ctx context.Context, dir string) bool { + if !settings.IsFilteredFetchesEnabled(ctx) { + return false + } + + repoDir, err := fetchWorkingDir(dir) + if err != nil { + return false + } + + shallow, err := isShallowRepositoryInDir(ctx, repoDir) + if err != nil { + return false + } + return shallow +} + +func fetchWorkingDir(dir string) (string, error) { + if dir != "" { + return filepath.Abs(dir) + } + return os.Getwd() +} + +func isShallowRepositoryInDir(ctx context.Context, dir string) (bool, error) { + cmd := exec.CommandContext(ctx, "git", "rev-parse", "--is-shallow-repository") + cmd.Dir = dir + disableTerminalPrompt(cmd) + out, err := cmd.Output() + if err != nil { + return false, fmt.Errorf("git rev-parse --is-shallow-repository: %w", err) + } + return strings.TrimSpace(string(out)) == "true", nil +} + // newCommand creates an exec.Cmd for a git operation that may need // checkpoint token authentication. If ENTIRE_CHECKPOINT_TOKEN is set: // - if the target in args is (or resolves to) an SSH remote, the target is diff --git a/cmd/entire/cli/checkpoint/remote/git_test.go b/cmd/entire/cli/checkpoint/remote/git_test.go index a6a9958224..4efdb46582 100644 --- a/cmd/entire/cli/checkpoint/remote/git_test.go +++ b/cmd/entire/cli/checkpoint/remote/git_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "os" "os/exec" + "path/filepath" "strings" "sync" "testing" @@ -256,6 +257,69 @@ func TestResolveFetchTarget(t *testing.T) { }) } +// Not parallel: uses t.Chdir() +func TestFetch_FilteredFetches_UnshallowsRepository(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + bareDir := filepath.Join(tmpDir, "bare.git") + seedDir := filepath.Join(tmpDir, "seed") + cloneDir := filepath.Join(tmpDir, "clone") + + testutil.InitRepo(t, seedDir) + testutil.WriteFile(t, seedDir, "f.txt", "init") + testutil.GitAdd(t, seedDir, "f.txt") + testutil.GitCommit(t, seedDir, "init") + + cmd := exec.CommandContext(ctx, "git", "init", "--bare", bareDir) + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run()) + + cmd = exec.CommandContext(ctx, "git", "remote", "add", "origin", bareDir) + cmd.Dir = seedDir + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run()) + + cmd = exec.CommandContext(ctx, "git", "push", "origin", "HEAD:refs/heads/main") + cmd.Dir = seedDir + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run()) + + cmd = exec.CommandContext(ctx, "git", "clone", "--depth=1", "--branch", "main", "file://"+bareDir, cloneDir) + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run()) + + // Advance the remote after the shallow clone so the subsequent fetch has + // new history to bring in while also deepening the repository. + testutil.WriteFile(t, seedDir, "f.txt", "init\nnext\n") + testutil.GitAdd(t, seedDir, "f.txt") + testutil.GitCommit(t, seedDir, "next") + cmd = exec.CommandContext(ctx, "git", "push", "origin", "HEAD:refs/heads/main") + cmd.Dir = seedDir + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run()) + + testutil.WriteFile( + t, + cloneDir, + ".entire/settings.json", + `{"enabled": true, "strategy_options": {"filtered_fetches": true}}`, + ) + + require.True(t, isShallowRepository(ctx, cloneDir), "test setup should produce a shallow repo") + + _, err := Fetch(ctx, FetchOptions{ + Remote: "file://" + bareDir, + RefSpecs: []string{"+refs/heads/main:refs/remotes/origin/main"}, + NoTags: true, + Dir: cloneDir, + }) + require.NoError(t, err) + + assert.False(t, isShallowRepository(ctx, cloneDir), + "filtered fetch should unshallow legacy shallow repositories") +} + func TestAppendCheckpointTokenEnv(t *testing.T) { t.Parallel() From 0738410b3d3fa7acb9d83d559496817e650c5ff7 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 24 Apr 2026 15:03:06 +0100 Subject: [PATCH 04/12] Disable auto-gc during fetches to reduce race risk Entire-Checkpoint: 3c18f86e29f8 --- cmd/entire/cli/checkpoint/remote/git.go | 2 +- cmd/entire/cli/checkpoint/remote/git_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/entire/cli/checkpoint/remote/git.go b/cmd/entire/cli/checkpoint/remote/git.go index 95f1f0f5cb..f889a5c1d4 100644 --- a/cmd/entire/cli/checkpoint/remote/git.go +++ b/cmd/entire/cli/checkpoint/remote/git.go @@ -48,7 +48,7 @@ type FetchOptions struct { // resolve the name to a URL (to avoid persisting promisor settings) should call // ResolveFetchTarget first and pass the resolved target as opts.Remote. func Fetch(ctx context.Context, opts FetchOptions) ([]byte, error) { - args := []string{"fetch"} + args := []string{"fetch", "--no-auto-gc"} if opts.NoTags { args = append(args, "--no-tags") } diff --git a/cmd/entire/cli/checkpoint/remote/git_test.go b/cmd/entire/cli/checkpoint/remote/git_test.go index 4efdb46582..cddfbcc444 100644 --- a/cmd/entire/cli/checkpoint/remote/git_test.go +++ b/cmd/entire/cli/checkpoint/remote/git_test.go @@ -28,10 +28,10 @@ func TestExtractRemoteFromArgs(t *testing.T) { args []string want string }{ - {"fetch with URL", []string{"fetch", "https://github.com/org/repo.git", "refs/heads/main"}, "https://github.com/org/repo.git"}, + {"fetch with URL", []string{"fetch", "--no-auto-gc", "https://github.com/org/repo.git", "refs/heads/main"}, "https://github.com/org/repo.git"}, {"push with flags", []string{"push", "--no-verify", "--porcelain", "origin", "main"}, "origin"}, {"ls-remote", []string{"ls-remote", "origin", "refs/heads/*"}, "origin"}, - {"fetch with filter", []string{"fetch", "--no-tags", "--filter=blob:none", "https://host/r.git", "+refs/heads/main:refs/tmp"}, "https://host/r.git"}, + {"fetch with filter", []string{"fetch", "--no-auto-gc", "--no-tags", "--filter=blob:none", "https://host/r.git", "+refs/heads/main:refs/tmp"}, "https://host/r.git"}, {"empty args", []string{}, ""}, {"subcommand only", []string{"fetch"}, ""}, {"only flags", []string{"fetch", "--no-tags"}, ""}, @@ -655,7 +655,7 @@ func TestNewCommand_GIT_TERMINAL_PROMPT_Coexistence(t *testing.T) { t.Setenv(CheckpointTokenEnvVar, "coexist-token") cmd := newCommand(context.Background(), - "fetch", "--no-tags", "--filter=blob:none", "https://github.com/org/repo.git", "refs/heads/main") + "fetch", "--no-auto-gc", "--no-tags", "--filter=blob:none", "https://github.com/org/repo.git", "refs/heads/main") require.NotNil(t, cmd.Env) cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0") From 62625cdec8066db1cebc28fc3eae6f648d68845e Mon Sep 17 00:00:00 2001 From: Stefan Haubold Date: Fri, 22 May 2026 17:41:44 +0200 Subject: [PATCH 05/12] Auto-unshallow any shallow checkpoint repo, not only filtered ones The previous gate (filtered_fetches AND shallow) left a class of users exposed: anyone whose repo became shallow for a non-Entire reason (a hand-run git clone --depth=N, an older CLI version that pre-dated the filtered-fetches setting, an unrelated tool) still saw the reconcile path walk parent chains past .git/shallow via go-git and propose to cherry-pick hundreds of stale commits onto the remote. Drop the gate. Any shallow checkpoint repo unshallows on the next fetch, regardless of how the shallow state was introduced. Test now covers both filtered_fetches on and off. Entire-Checkpoint: 5aacb2ba139e --- cmd/entire/cli/checkpoint/remote/git.go | 23 ++-- cmd/entire/cli/checkpoint/remote/git_test.go | 130 ++++++++++--------- 2 files changed, 82 insertions(+), 71 deletions(-) diff --git a/cmd/entire/cli/checkpoint/remote/git.go b/cmd/entire/cli/checkpoint/remote/git.go index f889a5c1d4..d1f44168ed 100644 --- a/cmd/entire/cli/checkpoint/remote/git.go +++ b/cmd/entire/cli/checkpoint/remote/git.go @@ -39,9 +39,11 @@ type FetchOptions struct { // Fetch runs git fetch with checkpoint token injection and optional // filtered fetches (--filter=blob:none when settings enable it). -// When filtered_fetches is enabled and the repository is shallow, Fetch also -// adds --unshallow. This migrates users away from legacy shallow checkpoint -// repositories because some Entire commands need checkpoint history. +// When the repository is shallow, Fetch also adds --unshallow. This migrates +// users away from any shallow checkpoint history (whether produced by an old +// CLI --depth=1 fetch or by an unrelated tool) because the reconcile and +// rebase paths walk parent chains via go-git, which ignores .git/shallow and +// would otherwise traverse stale objects past the shallow boundary. // GIT_TERMINAL_PROMPT=0 is always set. // // Callers that pass a remote name (e.g., "origin") and want filtered fetches to @@ -53,14 +55,13 @@ func Fetch(ctx context.Context, opts FetchOptions) ([]byte, error) { args = append(args, "--no-tags") } args = append(args, opts.ExtraArgs...) - filteredFetchesEnabled := settings.IsFilteredFetchesEnabled(ctx) - if filteredFetchesEnabled && isShallowRepository(ctx, opts.Dir) { - // Filtered checkpoint fetches used to create shallow repositories. - // Unshallow on subsequent filtered fetches so commands that rely on - // checkpoint ancestry/history can operate correctly. + if isShallowRepository(ctx, opts.Dir) { + // Unshallow whenever the repo is shallow so commands that rely on + // checkpoint ancestry/history can operate correctly, regardless of + // how the shallow state was introduced. args = append(args, "--unshallow") } - if !opts.NoFilter && filteredFetchesEnabled { + if !opts.NoFilter && settings.IsFilteredFetchesEnabled(ctx) { args = append(args, "--filter=blob:none") } args = append(args, opts.Remote) @@ -316,10 +317,6 @@ func ResolveFetchTarget(ctx context.Context, target string) (string, error) { } func isShallowRepository(ctx context.Context, dir string) bool { - if !settings.IsFilteredFetchesEnabled(ctx) { - return false - } - repoDir, err := fetchWorkingDir(dir) if err != nil { return false diff --git a/cmd/entire/cli/checkpoint/remote/git_test.go b/cmd/entire/cli/checkpoint/remote/git_test.go index cddfbcc444..d8211109f1 100644 --- a/cmd/entire/cli/checkpoint/remote/git_test.go +++ b/cmd/entire/cli/checkpoint/remote/git_test.go @@ -257,67 +257,81 @@ func TestResolveFetchTarget(t *testing.T) { }) } -// Not parallel: uses t.Chdir() -func TestFetch_FilteredFetches_UnshallowsRepository(t *testing.T) { - ctx := context.Background() - - tmpDir := t.TempDir() - bareDir := filepath.Join(tmpDir, "bare.git") - seedDir := filepath.Join(tmpDir, "seed") - cloneDir := filepath.Join(tmpDir, "clone") - - testutil.InitRepo(t, seedDir) - testutil.WriteFile(t, seedDir, "f.txt", "init") - testutil.GitAdd(t, seedDir, "f.txt") - testutil.GitCommit(t, seedDir, "init") - - cmd := exec.CommandContext(ctx, "git", "init", "--bare", bareDir) - cmd.Env = testutil.GitIsolatedEnv() - require.NoError(t, cmd.Run()) - - cmd = exec.CommandContext(ctx, "git", "remote", "add", "origin", bareDir) - cmd.Dir = seedDir - cmd.Env = testutil.GitIsolatedEnv() - require.NoError(t, cmd.Run()) - - cmd = exec.CommandContext(ctx, "git", "push", "origin", "HEAD:refs/heads/main") - cmd.Dir = seedDir - cmd.Env = testutil.GitIsolatedEnv() - require.NoError(t, cmd.Run()) - - cmd = exec.CommandContext(ctx, "git", "clone", "--depth=1", "--branch", "main", "file://"+bareDir, cloneDir) - cmd.Env = testutil.GitIsolatedEnv() - require.NoError(t, cmd.Run()) - - // Advance the remote after the shallow clone so the subsequent fetch has - // new history to bring in while also deepening the repository. - testutil.WriteFile(t, seedDir, "f.txt", "init\nnext\n") - testutil.GitAdd(t, seedDir, "f.txt") - testutil.GitCommit(t, seedDir, "next") - cmd = exec.CommandContext(ctx, "git", "push", "origin", "HEAD:refs/heads/main") - cmd.Dir = seedDir - cmd.Env = testutil.GitIsolatedEnv() - require.NoError(t, cmd.Run()) - - testutil.WriteFile( - t, - cloneDir, - ".entire/settings.json", - `{"enabled": true, "strategy_options": {"filtered_fetches": true}}`, - ) +func TestFetch_UnshallowsShallowRepository(t *testing.T) { + t.Parallel() - require.True(t, isShallowRepository(ctx, cloneDir), "test setup should produce a shallow repo") + for _, tc := range []struct { + name string + filteredFetches bool + settingsContents string + }{ + { + name: "filtered_fetches enabled", + settingsContents: `{"enabled": true, "strategy_options": {"filtered_fetches": true}}`, + }, + { + name: "filtered_fetches disabled", + settingsContents: `{"enabled": true}`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := context.Background() - _, err := Fetch(ctx, FetchOptions{ - Remote: "file://" + bareDir, - RefSpecs: []string{"+refs/heads/main:refs/remotes/origin/main"}, - NoTags: true, - Dir: cloneDir, - }) - require.NoError(t, err) + tmpDir := t.TempDir() + bareDir := filepath.Join(tmpDir, "bare.git") + seedDir := filepath.Join(tmpDir, "seed") + cloneDir := filepath.Join(tmpDir, "clone") + + testutil.InitRepo(t, seedDir) + testutil.WriteFile(t, seedDir, "f.txt", "init") + testutil.GitAdd(t, seedDir, "f.txt") + testutil.GitCommit(t, seedDir, "init") + + cmd := exec.CommandContext(ctx, "git", "init", "--bare", bareDir) + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run()) + + cmd = exec.CommandContext(ctx, "git", "remote", "add", "origin", bareDir) + cmd.Dir = seedDir + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run()) + + cmd = exec.CommandContext(ctx, "git", "push", "origin", "HEAD:refs/heads/main") + cmd.Dir = seedDir + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run()) + + cmd = exec.CommandContext(ctx, "git", "clone", "--depth=1", "--branch", "main", "file://"+bareDir, cloneDir) + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run()) + + // Advance the remote after the shallow clone so the subsequent fetch has + // new history to bring in while also deepening the repository. + testutil.WriteFile(t, seedDir, "f.txt", "init\nnext\n") + testutil.GitAdd(t, seedDir, "f.txt") + testutil.GitCommit(t, seedDir, "next") + cmd = exec.CommandContext(ctx, "git", "push", "origin", "HEAD:refs/heads/main") + cmd.Dir = seedDir + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run()) + + testutil.WriteFile(t, cloneDir, ".entire/settings.json", tc.settingsContents) + + require.True(t, isShallowRepository(ctx, cloneDir), "test setup should produce a shallow repo") + + _, err := Fetch(ctx, FetchOptions{ + Remote: "file://" + bareDir, + RefSpecs: []string{"+refs/heads/main:refs/remotes/origin/main"}, + NoTags: true, + Dir: cloneDir, + }) + require.NoError(t, err) - assert.False(t, isShallowRepository(ctx, cloneDir), - "filtered fetch should unshallow legacy shallow repositories") + assert.False(t, isShallowRepository(ctx, cloneDir), + "fetch should unshallow any shallow repository, regardless of filtered_fetches") + }) + } } func TestAppendCheckpointTokenEnv(t *testing.T) { From 63d77c05bc3f7cc1ad6a116d6894866471868ea5 Mon Sep 17 00:00:00 2001 From: Stefan Haubold Date: Fri, 22 May 2026 17:46:19 +0200 Subject: [PATCH 06/12] Respect .git/shallow when walking checkpoint history for reconcile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit collectCommitChain and the cherry-pick path used go-git to walk commit.ParentHashes. go-git does not consult .git/shallow, so on a shallow checkpoint repo the walk strolled past the boundary into stale pack objects — producing a phantom chain that the reconcile path then proposed to cherry-pick onto the remote tip, duplicating commits that already lived on the remote under their original SHAs. Read the shallow set once at the top of each reconcile/rebase entry point and pass it down. collectCommitChain stops at shallow commits; treeChangesForCherryPick treats shallow-boundary commits as roots and diffs them against an empty tree, so the cherry-pick contributes the commit's full content rather than a delta against a stale parent. This is independent of the auto-unshallow fix: a future shallow file introduced by any other tool (manual git clone --depth, an unrelated plumbing helper) would otherwise still trigger the same bug. Entire-Checkpoint: 49c40909c8f5 --- cmd/entire/cli/strategy/metadata_reconcile.go | 313 +++++++++++++++++- .../cli/strategy/metadata_reconcile_test.go | 93 +++++- cmd/entire/cli/strategy/push_common.go | 7 +- 3 files changed, 404 insertions(+), 9 deletions(-) diff --git a/cmd/entire/cli/strategy/metadata_reconcile.go b/cmd/entire/cli/strategy/metadata_reconcile.go index ccb616bb44..7fd645fdc3 100644 --- a/cmd/entire/cli/strategy/metadata_reconcile.go +++ b/cmd/entire/cli/strategy/metadata_reconcile.go @@ -8,10 +8,13 @@ import ( "log/slog" "os" "os/exec" + "path/filepath" + "strings" "sync" "time" "github.com/entireio/cli/cmd/entire/cli/checkpoint" + remote "github.com/entireio/cli/cmd/entire/cli/checkpoint/remote" "github.com/entireio/cli/cmd/entire/cli/logging" "github.com/entireio/cli/cmd/entire/cli/paths" @@ -151,8 +154,13 @@ func ReconcileDisconnectedMetadataBranch( // Disconnected — cherry-pick local commits onto remote tip fmt.Fprintln(w, "[entire] Detected disconnected session metadata (local and remote share no common ancestor)") + shallow, err := loadShallowHashes(ctx, repoPath) + if err != nil { + return fmt.Errorf("failed to load shallow boundaries: %w", err) + } + // Collect local commits oldest-first - localCommits, err := collectCommitChain(repo, localHash) + localCommits, err := collectCommitChain(repo, localHash, shallow) if err != nil { return fmt.Errorf("failed to collect local commits: %w", err) } @@ -181,7 +189,7 @@ func ReconcileDisconnectedMetadataBranch( fmt.Fprintf(w, "[entire] Cherry-picking %d local checkpoint(s) onto remote...\n", len(dataCommits)) - newTip, err := cherryPickOnto(ctx, repo, remoteHash, dataCommits) + newTip, err := cherryPickOnto(ctx, repo, remoteHash, dataCommits, shallow) if err != nil { return fmt.Errorf("failed to cherry-pick local commits onto remote: %w", err) } @@ -196,6 +204,243 @@ func ReconcileDisconnectedMetadataBranch( return nil } +// v2DoctorTmpRef is the temporary ref used by doctor to fetch and compare the remote v2 /main. +// Uses the refs/entire-fetch-tmp/ namespace consistent with checkpoint_remote.go. +const v2DoctorTmpRef = "refs/entire-fetch-tmp/doctor-v2-main" + +// IsV2MainDisconnected checks whether the local v2 /main ref and the remote +// v2 /main ref exist but share no common ancestor. Uses git ls-remote to +// discover the remote ref (custom refs don't have remote-tracking refs). +// +// remote is the git remote name, URL, or local path to check against. +// Returns (false, nil) if either ref doesn't exist or they share ancestry. +func IsV2MainDisconnected(ctx context.Context, repo *git.Repository, remote string) (bool, error) { + refName := plumbing.ReferenceName(paths.V2MainRefName) + + localRef, err := repo.Reference(refName, true) + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("failed to check local v2 /main ref: %w", err) + } + + repoPath, err := getRepoPath(repo) + if err != nil { + return false, err + } + + remoteHash, err := lsRemoteRef(ctx, repoPath, remote, paths.V2MainRefName) + if err != nil { + return false, fmt.Errorf("failed to ls-remote v2 /main: %w", err) + } + if remoteHash == plumbing.ZeroHash { + return false, nil // Remote doesn't have the ref + } + + if localRef.Hash() == remoteHash { + return false, nil + } + + // Fetch remote ref to temporary local ref for merge-base check. + // Use the fetched hash (not ls-remote hash) since the remote may have advanced. + if fetchErr := fetchRefToTemp(ctx, repoPath, remote, paths.V2MainRefName, v2DoctorTmpRef); fetchErr != nil { + return false, fmt.Errorf("failed to fetch remote v2 /main: %w", fetchErr) + } + defer cleanupTmpRef(repo) + + fetchedHash, err := resolveRefHash(repo, v2DoctorTmpRef) + if err != nil { + return false, fmt.Errorf("failed to read fetched v2 /main ref: %w", err) + } + + if localRef.Hash() == fetchedHash { + return false, nil + } + + return isDisconnected(ctx, repoPath, localRef.Hash().String(), fetchedHash.String()) +} + +// ReconcileDisconnectedV2Ref detects and repairs disconnected local/remote +// v2 /main refs. Same strategy as v1: cherry-pick local commits onto remote tip. +// The remote is discovered via git ls-remote and fetched to a temp ref. +// +// remote is the git remote name, URL, or local path. +func ReconcileDisconnectedV2Ref( + ctx context.Context, + repo *git.Repository, + remote string, + w io.Writer, +) error { + refName := plumbing.ReferenceName(paths.V2MainRefName) + + localRef, err := repo.Reference(refName, true) + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("failed to check local v2 /main ref: %w", err) + } + + repoPath, err := getRepoPath(repo) + if err != nil { + return err + } + + remoteHash, err := lsRemoteRef(ctx, repoPath, remote, paths.V2MainRefName) + if err != nil { + return fmt.Errorf("failed to ls-remote v2 /main: %w", err) + } + if remoteHash == plumbing.ZeroHash { + return nil + } + + if localRef.Hash() == remoteHash { + return nil + } + + if fetchErr := fetchRefToTemp(ctx, repoPath, remote, paths.V2MainRefName, v2DoctorTmpRef); fetchErr != nil { + return fmt.Errorf("failed to fetch remote v2 /main: %w", fetchErr) + } + defer cleanupTmpRef(repo) + + // Use the fetched hash (not ls-remote hash) since the remote may have advanced. + fetchedHash, err := resolveRefHash(repo, v2DoctorTmpRef) + if err != nil { + return fmt.Errorf("failed to read fetched v2 /main ref: %w", err) + } + + if localRef.Hash() == fetchedHash { + return nil + } + + disconnected, err := isDisconnected(ctx, repoPath, localRef.Hash().String(), fetchedHash.String()) + if err != nil { + return fmt.Errorf("failed to check v2 /main ancestry: %w", err) + } + if !disconnected { + return nil + } + + fmt.Fprintln(w, "[entire] Detected disconnected v2 /main refs (local and remote share no common ancestor)") + + shallow, err := loadShallowHashes(ctx, repoPath) + if err != nil { + return fmt.Errorf("failed to load shallow boundaries: %w", err) + } + + localCommits, err := collectCommitChain(repo, localRef.Hash(), shallow) + if err != nil { + return fmt.Errorf("failed to collect local commits: %w", err) + } + + var dataCommits []*object.Commit + for _, c := range localCommits { + tree, treeErr := c.Tree() + if treeErr != nil { + return fmt.Errorf("failed to read tree for commit %s: %w", c.Hash.String()[:7], treeErr) + } + if len(tree.Entries) > 0 { + dataCommits = append(dataCommits, c) + } + } + + if len(dataCommits) == 0 { + ref := plumbing.NewHashReference(refName, fetchedHash) + if setErr := repo.Storer.SetReference(ref); setErr != nil { + return fmt.Errorf("failed to reset v2 /main to remote: %w", setErr) + } + fmt.Fprintln(w, "[entire] Done — local had no checkpoint data, reset to remote") + return nil + } + + fmt.Fprintf(w, "[entire] Cherry-picking %d local checkpoint(s) onto remote...\n", len(dataCommits)) + + newTip, err := cherryPickOnto(ctx, repo, fetchedHash, dataCommits, shallow) + if err != nil { + return fmt.Errorf("failed to cherry-pick local commits onto remote: %w", err) + } + + ref := plumbing.NewHashReference(refName, newTip) + if setErr := repo.Storer.SetReference(ref); setErr != nil { + return fmt.Errorf("failed to update v2 /main ref: %w", setErr) + } + + fmt.Fprintln(w, "[entire] Done — all local and remote checkpoints preserved") + return nil +} + +// lsRemoteRef runs git ls-remote and returns the hash for a specific ref. +// Returns plumbing.ZeroHash if the ref doesn't exist on the remote. +func lsRemoteRef(ctx context.Context, repoPath, remoteName, refName string) (plumbing.Hash, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + fetchTarget, err := remote.ResolveFetchTarget(ctx, remoteName) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("resolve fetch target for ls-remote: %w", err) + } + + output, err := remote.LsRemoteInDir(ctx, repoPath, fetchTarget, refName) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("git ls-remote %s failed: %w", remote.RedactURL(fetchTarget), err) + } + + line := strings.TrimSpace(string(output)) + if line == "" { + return plumbing.ZeroHash, nil + } + + parts := strings.Fields(line) + if len(parts) < 2 { + return plumbing.ZeroHash, nil + } + + return plumbing.NewHash(parts[0]), nil +} + +// fetchRefToTemp fetches a remote ref to a temporary local ref for comparison. +func fetchRefToTemp(ctx context.Context, repoPath, remoteName, srcRef, dstRef string) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + fetchTarget, err := remote.ResolveFetchTarget(ctx, remoteName) + if err != nil { + return fmt.Errorf("resolve fetch target for doctor v2 fetch: %w", err) + } + + refspec := fmt.Sprintf("+%s:%s", srcRef, dstRef) + output, err := remote.Fetch(ctx, remote.FetchOptions{ + Remote: fetchTarget, + RefSpecs: []string{refspec}, + NoTags: true, + Dir: repoPath, + }) + if err != nil { + redactedURL := remote.RedactURL(fetchTarget) + msg := strings.TrimSpace(strings.ReplaceAll(string(output), fetchTarget, redactedURL)) + if msg != "" { + return fmt.Errorf("git fetch %s failed: %s: %w", redactedURL, msg, err) + } + return fmt.Errorf("git fetch %s failed: %w", redactedURL, err) + } + return nil +} + +// resolveRefHash reads the commit hash that a ref points to. +func resolveRefHash(repo *git.Repository, refName string) (plumbing.Hash, error) { + ref, err := repo.Reference(plumbing.ReferenceName(refName), true) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("ref %s not found: %w", refName, err) + } + return ref.Hash(), nil +} + +// cleanupTmpRef deletes the temporary ref used by doctor checks. +func cleanupTmpRef(repo *git.Repository) { + _ = repo.Storer.RemoveReference(plumbing.ReferenceName(v2DoctorTmpRef)) //nolint:errcheck // best-effort cleanup +} + // isDisconnected checks if two commits have no common ancestor using git merge-base. // Returns (true, nil) if disconnected, (false, nil) if they share ancestry, // or (false, error) if git merge-base failed for another reason. @@ -220,7 +465,13 @@ func isDisconnected(ctx context.Context, repoPath, hashA, hashB string) (bool, e } // collectCommitChain walks from tip to root following first parent, returns oldest-first. -func collectCommitChain(repo *git.Repository, tip plumbing.Hash) ([]*object.Commit, error) { +// Commits listed in shallow are treated as roots — the walk stops at them without +// traversing into their parents. go-git's repo.CommitObject().ParentHashes does not +// consult .git/shallow on its own, so without this check the walk would stroll past +// shallow boundaries into stale objects left in the pack (e.g., when the remote +// branch has been rebuilt since the last full fetch), producing a phantom chain of +// commits that no longer represent the actual checkpoint history. +func collectCommitChain(repo *git.Repository, tip plumbing.Hash, shallow map[plumbing.Hash]bool) ([]*object.Commit, error) { var chain []*object.Commit current := tip @@ -236,6 +487,11 @@ func collectCommitChain(repo *git.Repository, tip plumbing.Hash) ([]*object.Comm reachedRoot = true break } + if shallow[current] { + // Shallow boundary — treat as a root. + reachedRoot = true + break + } current = commit.ParentHashes[0] } @@ -251,14 +507,53 @@ func collectCommitChain(repo *git.Repository, tip plumbing.Hash) ([]*object.Comm return chain, nil } +// loadShallowHashes returns the set of commit hashes listed in the repository's +// shallow file (one hash per line). For a non-shallow repository, returns an +// empty (non-nil) map. The shallow file lives under the git common dir, so this +// works for linked worktrees as well as the primary worktree. +func loadShallowHashes(ctx context.Context, repoPath string) (map[plumbing.Hash]bool, error) { + cmd := exec.CommandContext(ctx, "git", "rev-parse", "--git-common-dir") + cmd.Dir = repoPath + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git rev-parse --git-common-dir: %w", err) + } + gitDir := strings.TrimSpace(string(out)) + if !filepath.IsAbs(gitDir) { + gitDir = filepath.Join(repoPath, gitDir) + } + data, err := os.ReadFile(filepath.Join(gitDir, "shallow")) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return map[plumbing.Hash]bool{}, nil + } + return nil, fmt.Errorf("read shallow file: %w", err) + } + set := map[plumbing.Hash]bool{} + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + set[plumbing.NewHash(line)] = true + } + return set, nil +} + // cherryPickOnto applies each commit's delta onto base, building a linear chain. // For each commit, it computes the full diff from its parent (additions, modifications, // and deletions), then applies that delta onto the current tip's tree. -func cherryPickOnto(ctx context.Context, repo *git.Repository, base plumbing.Hash, commits []*object.Commit) (plumbing.Hash, error) { +// +// Commits listed in shallow are treated as roots: their delta is computed against +// an empty tree rather than against their (past-the-boundary) parent. Without this, +// a shallow-boundary commit would be diffed against a stale parent tree whose +// objects live in the local pack but no longer represent the actual checkpoint +// history — producing nonsense changes when replayed onto the remote tip. +func cherryPickOnto(ctx context.Context, repo *git.Repository, base plumbing.Hash, commits []*object.Commit, shallow map[plumbing.Hash]bool) (plumbing.Hash, error) { currentTip := base for _, commit := range commits { - changes, err := treeChangesForCherryPick(ctx, repo, commit) + changes, err := treeChangesForCherryPick(ctx, repo, commit, shallow) if err != nil { return plumbing.ZeroHash, err } @@ -288,14 +583,18 @@ func cherryPickOnto(ctx context.Context, repo *git.Repository, base plumbing.Has return currentTip, nil } -func treeChangesForCherryPick(ctx context.Context, repo *git.Repository, commit *object.Commit) ([]checkpoint.TreeChange, error) { +func treeChangesForCherryPick(ctx context.Context, repo *git.Repository, commit *object.Commit, shallow map[plumbing.Hash]bool) ([]checkpoint.TreeChange, error) { commitTree, err := commit.Tree() if err != nil { return nil, fmt.Errorf("failed to get tree for commit %s: %w", commit.Hash, err) } var parentTree *object.Tree - if len(commit.ParentHashes) > 0 { + // Skip parent lookup for shallow-boundary commits: their stored ParentHashes + // point past the boundary into objects we may have but that no longer + // represent the actual checkpoint history. Treat them as roots — diff against + // an empty tree so the cherry-pick contributes the commit's full content. + if len(commit.ParentHashes) > 0 && !shallow[commit.Hash] { parentCommit, pErr := repo.CommitObject(commit.ParentHashes[0]) if pErr != nil { return nil, fmt.Errorf("failed to get parent commit %s: %w", commit.ParentHashes[0], pErr) diff --git a/cmd/entire/cli/strategy/metadata_reconcile_test.go b/cmd/entire/cli/strategy/metadata_reconcile_test.go index b98fa2c554..97420615bc 100644 --- a/cmd/entire/cli/strategy/metadata_reconcile_test.go +++ b/cmd/entire/cli/strategy/metadata_reconcile_test.go @@ -665,12 +665,103 @@ func TestCollectCommitChain_DepthLimit(t *testing.T) { tip = h } - _, err = collectCommitChain(repo, tip) + _, err = collectCommitChain(repo, tip, nil) require.Error(t, err) assert.Contains(t, err.Error(), "exceeded") assert.Contains(t, err.Error(), "without reaching root") } +// TestCollectCommitChain_StopsAtShallowBoundary verifies that collectCommitChain +// treats commits listed in the shallow set as roots, stopping the walk at the +// boundary even when the boundary commit has a parent SHA recorded in the object +// store. Without this behaviour, a shallow checkpoint repo whose remote v1 was +// rebuilt elsewhere would produce a phantom chain of stale commits. +func TestCollectCommitChain_StopsAtShallowBoundary(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + repo, err := git.PlainInit(dir, false) + require.NoError(t, err) + + emptyTree := &object.Tree{Entries: []object.TreeEntry{}} + treeObj := repo.Storer.NewEncodedObject() + require.NoError(t, emptyTree.Encode(treeObj)) + treeHash, err := repo.Storer.SetEncodedObject(treeObj) + require.NoError(t, err) + + // Build a linear chain of 10 commits. Without shallow, the walk should + // return all 10. With the 4th from the tip marked shallow, it should stop + // at that commit, returning 4 entries (tip + 3 below it, including the + // shallow boundary itself, treated as a root). + var tip plumbing.Hash + hashes := make([]plumbing.Hash, 0, 10) + for i := range 10 { + c := &object.Commit{ + TreeHash: treeHash, + Author: object.Signature{Name: "test", Email: "test@test.com", When: time.Now().Add(time.Duration(i) * time.Second)}, + Committer: object.Signature{Name: "test", Email: "test@test.com", When: time.Now().Add(time.Duration(i) * time.Second)}, + Message: "commit\n", + } + if tip != plumbing.ZeroHash { + c.ParentHashes = []plumbing.Hash{tip} + } + obj := repo.Storer.NewEncodedObject() + require.NoError(t, c.Encode(obj)) + h, sErr := repo.Storer.SetEncodedObject(obj) + require.NoError(t, sErr) + hashes = append(hashes, h) + tip = h + } + + // Without shallow set: full chain of 10. + chain, err := collectCommitChain(repo, tip, nil) + require.NoError(t, err) + assert.Len(t, chain, 10, "without shallow, expect full chain") + + // With the 4th-from-tip (index 6 in build order) marked shallow: walk + // stops there. Result is oldest-first: shallow boundary, then up to tip + // = 4 commits. + shallow := map[plumbing.Hash]bool{hashes[6]: true} + chain, err = collectCommitChain(repo, tip, shallow) + require.NoError(t, err) + require.Len(t, chain, 4, "expect tip + 3 commits down to the shallow boundary inclusive") + assert.Equal(t, hashes[6], chain[0].Hash, "oldest entry should be the shallow boundary") + assert.Equal(t, tip, chain[3].Hash, "newest entry should be the tip") +} + +func TestLoadShallowHashes(t *testing.T) { + t.Parallel() + + t.Run("non-shallow repo returns empty set", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + _, err := git.PlainInit(dir, false) + require.NoError(t, err) + + set, err := loadShallowHashes(context.Background(), dir) + require.NoError(t, err) + assert.Empty(t, set) + }) + + t.Run("reads .git/shallow hash list", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + _, err := git.PlainInit(dir, false) + require.NoError(t, err) + + shallowFile := filepath.Join(dir, ".git", "shallow") + require.NoError(t, os.WriteFile(shallowFile, + []byte("be156aa7cc38c2c6117246cb8adad068a3886351\n82b2a8554cccd19f5aa60f520f645b32d8bc2400\n"), + 0o644)) + + set, err := loadShallowHashes(context.Background(), dir) + require.NoError(t, err) + assert.Len(t, set, 2) + assert.True(t, set[plumbing.NewHash("be156aa7cc38c2c6117246cb8adad068a3886351")]) + assert.True(t, set[plumbing.NewHash("82b2a8554cccd19f5aa60f520f645b32d8bc2400")]) + }) +} + // TestReconcileDisconnected_AllEmptyOrphans verifies that when all local commits // are empty-tree orphan commits (the exact bug artifact), reconciliation resets // the local branch to the remote tip without cherry-picking. diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index a5f2e91b7b..594219f1a6 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -413,7 +413,12 @@ func fetchAndRebaseSessionsCommon(ctx context.Context, target, branchName string return nil } - newTip, err := cherryPickOnto(ctx, repo, remoteRef.Hash(), localCommits) + shallow, err := loadShallowHashes(ctx, repoPath) + if err != nil { + return fmt.Errorf("failed to load shallow boundaries: %w", err) + } + + newTip, err := cherryPickOnto(ctx, repo, remoteRef.Hash(), localCommits, shallow) if err != nil { return fmt.Errorf("failed to rebase local commits onto remote: %w", err) } From 2c0a5fc50369f7ae200fa3f15a22e17edc7f6527 Mon Sep 17 00:00:00 2001 From: Stefan Haubold Date: Fri, 22 May 2026 17:48:24 +0200 Subject: [PATCH 07/12] Satisfy lint: wrap errors and annotate intentional os.Getwd / os.ReadFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fetchWorkingDir: wrap filepath.Abs error, document why os.Getwd fallback is intentional (caller passed empty Dir to mean "inherit cwd"; tests rely on this), nolint forbidigo with reason. - loadShallowHashes: nolint gosec G304 — path is derived from git's own --git-common-dir output, not user input. No behavior change. Entire-Checkpoint: b5bbff32a63d --- cmd/entire/cli/checkpoint/remote/git.go | 16 ++++++++++++++-- cmd/entire/cli/strategy/metadata_reconcile.go | 3 ++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/cmd/entire/cli/checkpoint/remote/git.go b/cmd/entire/cli/checkpoint/remote/git.go index d1f44168ed..8d1d3b442f 100644 --- a/cmd/entire/cli/checkpoint/remote/git.go +++ b/cmd/entire/cli/checkpoint/remote/git.go @@ -331,9 +331,21 @@ func isShallowRepository(ctx context.Context, dir string) bool { func fetchWorkingDir(dir string) (string, error) { if dir != "" { - return filepath.Abs(dir) + abs, err := filepath.Abs(dir) + if err != nil { + return "", fmt.Errorf("resolve absolute path for %q: %w", dir, err) + } + return abs, nil + } + // Falling back to os.Getwd is intentional: the caller passed an empty Dir, + // meaning "use the inherited working directory of the git invocation". + // paths.RepoRoot() would impose a worktree-relative resolution that isn't + // always correct for fetch helpers called outside a repo context (tests). + cwd, err := os.Getwd() //nolint:forbidigo // see comment above + if err != nil { + return "", fmt.Errorf("get working dir: %w", err) } - return os.Getwd() + return cwd, nil } func isShallowRepositoryInDir(ctx context.Context, dir string) (bool, error) { diff --git a/cmd/entire/cli/strategy/metadata_reconcile.go b/cmd/entire/cli/strategy/metadata_reconcile.go index 7fd645fdc3..82e9052983 100644 --- a/cmd/entire/cli/strategy/metadata_reconcile.go +++ b/cmd/entire/cli/strategy/metadata_reconcile.go @@ -522,7 +522,8 @@ func loadShallowHashes(ctx context.Context, repoPath string) (map[plumbing.Hash] if !filepath.IsAbs(gitDir) { gitDir = filepath.Join(repoPath, gitDir) } - data, err := os.ReadFile(filepath.Join(gitDir, "shallow")) + // Path is constructed from git's own --git-common-dir output, not user input. + data, err := os.ReadFile(filepath.Join(gitDir, "shallow")) //nolint:gosec // see comment above if err != nil { if errors.Is(err, os.ErrNotExist) { return map[plumbing.Hash]bool{}, nil From 8f5072561eedf5dc2c897721a6af3eb3f243a8b9 Mon Sep 17 00:00:00 2001 From: Stefan Haubold Date: Fri, 22 May 2026 17:56:55 +0200 Subject: [PATCH 08/12] Always fetch-first in metadata resolution, even when filtered_fetches is off The previous gating in getMetadataTree / getV2MetadataTree skipped the tree-only fetch entirely when filtered_fetches was disabled. With no fetch step, the local ref lookup fell through to whatever the worktree last saw, which could be stale (a collaborator pushed a newer checkpoint, or our last fetch was before the producer published). The "full fetch" fallback then never ran because the stale local lookup "succeeded." Replace the gate with a first-attempt fetch that is always issued: tree-only when filtered_fetches is enabled (cheaper), full fetch otherwise. For v2 the secondary fallback is set to nil when the first-attempt is already the full fetch, to avoid a duplicate network round-trip. Restores the pre-PR property that a fetch always precedes the local lookup. Regression covered by TestRunExplainExport_JSONFetchesRemoteV2WhenLocalV2RefSelectsDualMode. Entire-Checkpoint: 60d8c085123a --- cmd/entire/cli/resume.go | 70 +++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/cmd/entire/cli/resume.go b/cmd/entire/cli/resume.go index 695c79136c..aec2b1f982 100644 --- a/cmd/entire/cli/resume.go +++ b/cmd/entire/cli/resume.go @@ -456,30 +456,40 @@ func getMetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) ) } - // Only use the tree-only fetch path when filtered fetches are enabled. - // Otherwise this would degrade into an ordinary full fetch while the - // surrounding control flow still assumes the tree-only fast path. - if settings.IsFilteredFetchesEnabled(ctx) { - if fetchErr := FetchMetadataTreeOnly(ctx); fetchErr == nil { - freshRepo, repoErr := openRepository(ctx) - if repoErr == nil { - logRefHash(freshRepo, "treeless-fetch") - metadataTree, treeErr := strategy.GetMetadataBranchTree(freshRepo) - if treeErr == nil { - logging.Debug(logCtx, "metadata tree obtained via treeless fetch", - slog.String("tree_hash", metadataTree.Hash.String()), - ) - return metadataTree, freshRepo, nil - } - logging.Debug(logCtx, "treeless fetch succeeded but tree read failed", - slog.String("error", treeErr.Error()), + // First-attempt fetch ensures we have fresh remote data before trusting + // the local ref. Use the tree-only fetch when filtered_fetches is enabled + // (cheaper); otherwise it would degenerate into a full fetch, so call + // FetchMetadataBranch directly. Either way, fetch first so the local + // lookup below doesn't return stale data when a collaborator pushed new + // checkpoints since our last fetch. + firstFetch := FetchMetadataTreeOnly + firstFetchLabel := "treeless-fetch" + if !settings.IsFilteredFetchesEnabled(ctx) { + firstFetch = FetchMetadataBranch + firstFetchLabel = "full-fetch" + } + if fetchErr := firstFetch(ctx); fetchErr == nil { + freshRepo, repoErr := openRepository(ctx) + if repoErr == nil { + logRefHash(freshRepo, firstFetchLabel) + metadataTree, treeErr := strategy.GetMetadataBranchTree(freshRepo) + if treeErr == nil { + logging.Debug(logCtx, "metadata tree obtained via first-attempt fetch", + slog.String("tree_hash", metadataTree.Hash.String()), + slog.String("label", firstFetchLabel), ) + return metadataTree, freshRepo, nil } - } else { - logging.Debug(logCtx, "treeless fetch failed, trying local", - slog.String("error", fetchErr.Error()), + logging.Debug(logCtx, "first-attempt fetch succeeded but tree read failed", + slog.String("error", treeErr.Error()), + slog.String("label", firstFetchLabel), ) } + } else { + logging.Debug(logCtx, "first-attempt fetch failed, trying local", + slog.String("error", fetchErr.Error()), + slog.String("label", firstFetchLabel), + ) } // Try local (may have been set by a prior fetch or push) @@ -541,12 +551,20 @@ func getMetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) // getV2MetadataTree resolves the v2 /main ref tree with the same // fetch fallback pattern as getMetadataTree, including checkpoint remote support. func getV2MetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) { - var treeOnlyFetch checkpoint.FetchRefFunc - if settings.IsFilteredFetchesEnabled(ctx) { - treeOnlyFetch = FetchV2MainTreeOnly - } - - tree, repo, err := checkpoint.GetV2MetadataTree(ctx, treeOnlyFetch, FetchV2MainRef, openRepository) + // First-attempt fetch ensures we have fresh remote data before trusting + // the local ref (which may be stale, e.g., a collaborator pushed a new + // checkpoint since the last fetch). When filtered_fetches is enabled the + // tree-only fetch is the cheaper option; otherwise it would degenerate + // into a full fetch, so call the full-fetch helper directly and pass nil + // as the secondary fallback to avoid a duplicate network round-trip. + firstFetch := checkpoint.FetchRefFunc(FetchV2MainTreeOnly) + secondFetch := checkpoint.FetchRefFunc(FetchV2MainRef) + if !settings.IsFilteredFetchesEnabled(ctx) { + firstFetch = FetchV2MainRef + secondFetch = nil + } + + tree, repo, err := checkpoint.GetV2MetadataTree(ctx, firstFetch, secondFetch, openRepository) if err == nil { return tree, repo, nil } From 90a2cf7a4d233ac9ea11a9d7162cf0797b02a2ff Mon Sep 17 00:00:00 2001 From: Stefan Haubold Date: Fri, 22 May 2026 18:18:57 +0200 Subject: [PATCH 09/12] Address code review: trim comments, simplify isShallowRepository, factor test setup - Drop fetchWorkingDir/isShallowRepositoryInDir indirection in isShallowRepository: cmd.Dir = "" already inherits parent cwd, so we don't need to resolve it ourselves. Removes the os.Getwd nolint and the filepath.Abs wrap. - Trim multi-paragraph doc on loadShallowHashes and the inline shallow rationale in treeChangesForCherryPick (cherryPickOnto's doc already explains the WHY). - Resume.go: collapse paired firstFetch/firstFetchLabel assignments into one line each; trim over-explanatory "First-attempt fetch ensures..." block in getV2MetadataTree. - Factor TestFetch_UnshallowsShallowRepository's setup into setupShallowClone + runIsolatedGit helpers so the table body is now the actual assertion rather than 70 lines of duplicated scaffolding. Entire-Checkpoint: cd78c8c0f7bd --- cmd/entire/cli/checkpoint/remote/git.go | 39 ++------ cmd/entire/cli/checkpoint/remote/git_test.go | 90 +++++++++---------- cmd/entire/cli/resume.go | 26 +++--- cmd/entire/cli/strategy/metadata_reconcile.go | 11 +-- 4 files changed, 60 insertions(+), 106 deletions(-) diff --git a/cmd/entire/cli/checkpoint/remote/git.go b/cmd/entire/cli/checkpoint/remote/git.go index 8d1d3b442f..28281a875d 100644 --- a/cmd/entire/cli/checkpoint/remote/git.go +++ b/cmd/entire/cli/checkpoint/remote/git.go @@ -316,47 +316,18 @@ func ResolveFetchTarget(ctx context.Context, target string) (string, error) { return url, nil } +// isShallowRepository returns true when the git repository at dir is shallow. +// An empty dir inherits the parent process's working directory, matching the +// semantics callers use when invoking Fetch with empty FetchOptions.Dir. func isShallowRepository(ctx context.Context, dir string) bool { - repoDir, err := fetchWorkingDir(dir) - if err != nil { - return false - } - - shallow, err := isShallowRepositoryInDir(ctx, repoDir) - if err != nil { - return false - } - return shallow -} - -func fetchWorkingDir(dir string) (string, error) { - if dir != "" { - abs, err := filepath.Abs(dir) - if err != nil { - return "", fmt.Errorf("resolve absolute path for %q: %w", dir, err) - } - return abs, nil - } - // Falling back to os.Getwd is intentional: the caller passed an empty Dir, - // meaning "use the inherited working directory of the git invocation". - // paths.RepoRoot() would impose a worktree-relative resolution that isn't - // always correct for fetch helpers called outside a repo context (tests). - cwd, err := os.Getwd() //nolint:forbidigo // see comment above - if err != nil { - return "", fmt.Errorf("get working dir: %w", err) - } - return cwd, nil -} - -func isShallowRepositoryInDir(ctx context.Context, dir string) (bool, error) { cmd := exec.CommandContext(ctx, "git", "rev-parse", "--is-shallow-repository") cmd.Dir = dir disableTerminalPrompt(cmd) out, err := cmd.Output() if err != nil { - return false, fmt.Errorf("git rev-parse --is-shallow-repository: %w", err) + return false } - return strings.TrimSpace(string(out)) == "true", nil + return strings.TrimSpace(string(out)) == "true" } // newCommand creates an exec.Cmd for a git operation that may need diff --git a/cmd/entire/cli/checkpoint/remote/git_test.go b/cmd/entire/cli/checkpoint/remote/git_test.go index d8211109f1..061672389f 100644 --- a/cmd/entire/cli/checkpoint/remote/git_test.go +++ b/cmd/entire/cli/checkpoint/remote/git_test.go @@ -262,62 +262,17 @@ func TestFetch_UnshallowsShallowRepository(t *testing.T) { for _, tc := range []struct { name string - filteredFetches bool settingsContents string }{ - { - name: "filtered_fetches enabled", - settingsContents: `{"enabled": true, "strategy_options": {"filtered_fetches": true}}`, - }, - { - name: "filtered_fetches disabled", - settingsContents: `{"enabled": true}`, - }, + {"filtered_fetches enabled", `{"enabled": true, "strategy_options": {"filtered_fetches": true}}`}, + {"filtered_fetches disabled", `{"enabled": true}`}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() ctx := context.Background() - tmpDir := t.TempDir() - bareDir := filepath.Join(tmpDir, "bare.git") - seedDir := filepath.Join(tmpDir, "seed") - cloneDir := filepath.Join(tmpDir, "clone") - - testutil.InitRepo(t, seedDir) - testutil.WriteFile(t, seedDir, "f.txt", "init") - testutil.GitAdd(t, seedDir, "f.txt") - testutil.GitCommit(t, seedDir, "init") - - cmd := exec.CommandContext(ctx, "git", "init", "--bare", bareDir) - cmd.Env = testutil.GitIsolatedEnv() - require.NoError(t, cmd.Run()) - - cmd = exec.CommandContext(ctx, "git", "remote", "add", "origin", bareDir) - cmd.Dir = seedDir - cmd.Env = testutil.GitIsolatedEnv() - require.NoError(t, cmd.Run()) - - cmd = exec.CommandContext(ctx, "git", "push", "origin", "HEAD:refs/heads/main") - cmd.Dir = seedDir - cmd.Env = testutil.GitIsolatedEnv() - require.NoError(t, cmd.Run()) - - cmd = exec.CommandContext(ctx, "git", "clone", "--depth=1", "--branch", "main", "file://"+bareDir, cloneDir) - cmd.Env = testutil.GitIsolatedEnv() - require.NoError(t, cmd.Run()) - - // Advance the remote after the shallow clone so the subsequent fetch has - // new history to bring in while also deepening the repository. - testutil.WriteFile(t, seedDir, "f.txt", "init\nnext\n") - testutil.GitAdd(t, seedDir, "f.txt") - testutil.GitCommit(t, seedDir, "next") - cmd = exec.CommandContext(ctx, "git", "push", "origin", "HEAD:refs/heads/main") - cmd.Dir = seedDir - cmd.Env = testutil.GitIsolatedEnv() - require.NoError(t, cmd.Run()) - + bareDir, cloneDir := setupShallowClone(ctx, t) testutil.WriteFile(t, cloneDir, ".entire/settings.json", tc.settingsContents) - require.True(t, isShallowRepository(ctx, cloneDir), "test setup should produce a shallow repo") _, err := Fetch(ctx, FetchOptions{ @@ -334,6 +289,45 @@ func TestFetch_UnshallowsShallowRepository(t *testing.T) { } } +// setupShallowClone creates a bare origin, a seed repo with one commit pushed +// to it, a shallow (--depth=1) clone, and then advances origin by one more +// commit so that a subsequent fetch into the clone has work to do. Returns the +// bare origin path and the shallow clone path. +func setupShallowClone(ctx context.Context, t *testing.T) (bareDir, cloneDir string) { + t.Helper() + tmpDir := t.TempDir() + bareDir = filepath.Join(tmpDir, "bare.git") + seedDir := filepath.Join(tmpDir, "seed") + cloneDir = filepath.Join(tmpDir, "clone") + + testutil.InitRepo(t, seedDir) + testutil.WriteFile(t, seedDir, "f.txt", "init") + testutil.GitAdd(t, seedDir, "f.txt") + testutil.GitCommit(t, seedDir, "init") + + runIsolatedGit(ctx, t, "", "init", "--bare", bareDir) + runIsolatedGit(ctx, t, seedDir, "remote", "add", "origin", bareDir) + runIsolatedGit(ctx, t, seedDir, "push", "origin", "HEAD:refs/heads/main") + runIsolatedGit(ctx, t, "", "clone", "--depth=1", "--branch", "main", "file://"+bareDir, cloneDir) + + testutil.WriteFile(t, seedDir, "f.txt", "init\nnext\n") + testutil.GitAdd(t, seedDir, "f.txt") + testutil.GitCommit(t, seedDir, "next") + runIsolatedGit(ctx, t, seedDir, "push", "origin", "HEAD:refs/heads/main") + + return bareDir, cloneDir +} + +func runIsolatedGit(ctx context.Context, t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.CommandContext(ctx, "git", args...) + if dir != "" { + cmd.Dir = dir + } + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run(), "git %v", args) +} + func TestAppendCheckpointTokenEnv(t *testing.T) { t.Parallel() diff --git a/cmd/entire/cli/resume.go b/cmd/entire/cli/resume.go index aec2b1f982..b5c4328df4 100644 --- a/cmd/entire/cli/resume.go +++ b/cmd/entire/cli/resume.go @@ -456,17 +456,13 @@ func getMetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) ) } - // First-attempt fetch ensures we have fresh remote data before trusting - // the local ref. Use the tree-only fetch when filtered_fetches is enabled - // (cheaper); otherwise it would degenerate into a full fetch, so call - // FetchMetadataBranch directly. Either way, fetch first so the local - // lookup below doesn't return stale data when a collaborator pushed new - // checkpoints since our last fetch. - firstFetch := FetchMetadataTreeOnly - firstFetchLabel := "treeless-fetch" + // Fetch first so the local lookup below doesn't return stale data when a + // collaborator pushed new checkpoints since our last fetch. Tree-only when + // filtered_fetches is on (cheaper); full fetch otherwise (tree-only would + // degenerate into a full fetch anyway). + firstFetch, firstFetchLabel := FetchMetadataTreeOnly, "treeless-fetch" if !settings.IsFilteredFetchesEnabled(ctx) { - firstFetch = FetchMetadataBranch - firstFetchLabel = "full-fetch" + firstFetch, firstFetchLabel = FetchMetadataBranch, "full-fetch" } if fetchErr := firstFetch(ctx); fetchErr == nil { freshRepo, repoErr := openRepository(ctx) @@ -551,12 +547,10 @@ func getMetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) // getV2MetadataTree resolves the v2 /main ref tree with the same // fetch fallback pattern as getMetadataTree, including checkpoint remote support. func getV2MetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) { - // First-attempt fetch ensures we have fresh remote data before trusting - // the local ref (which may be stale, e.g., a collaborator pushed a new - // checkpoint since the last fetch). When filtered_fetches is enabled the - // tree-only fetch is the cheaper option; otherwise it would degenerate - // into a full fetch, so call the full-fetch helper directly and pass nil - // as the secondary fallback to avoid a duplicate network round-trip. + // Fetch first so the local lookup doesn't return stale data. When + // filtered_fetches is off, use the full fetch directly (tree-only would + // degenerate into the same call) and pass nil as the fallback to avoid a + // duplicate round-trip. firstFetch := checkpoint.FetchRefFunc(FetchV2MainTreeOnly) secondFetch := checkpoint.FetchRefFunc(FetchV2MainRef) if !settings.IsFilteredFetchesEnabled(ctx) { diff --git a/cmd/entire/cli/strategy/metadata_reconcile.go b/cmd/entire/cli/strategy/metadata_reconcile.go index 82e9052983..fc01013371 100644 --- a/cmd/entire/cli/strategy/metadata_reconcile.go +++ b/cmd/entire/cli/strategy/metadata_reconcile.go @@ -507,10 +507,8 @@ func collectCommitChain(repo *git.Repository, tip plumbing.Hash, shallow map[plu return chain, nil } -// loadShallowHashes returns the set of commit hashes listed in the repository's -// shallow file (one hash per line). For a non-shallow repository, returns an -// empty (non-nil) map. The shallow file lives under the git common dir, so this -// works for linked worktrees as well as the primary worktree. +// loadShallowHashes returns the commit hashes listed in the repository's +// shallow file, or an empty map if the repository is not shallow. func loadShallowHashes(ctx context.Context, repoPath string) (map[plumbing.Hash]bool, error) { cmd := exec.CommandContext(ctx, "git", "rev-parse", "--git-common-dir") cmd.Dir = repoPath @@ -591,10 +589,7 @@ func treeChangesForCherryPick(ctx context.Context, repo *git.Repository, commit } var parentTree *object.Tree - // Skip parent lookup for shallow-boundary commits: their stored ParentHashes - // point past the boundary into objects we may have but that no longer - // represent the actual checkpoint history. Treat them as roots — diff against - // an empty tree so the cherry-pick contributes the commit's full content. + // Shallow-boundary commits are treated as roots — see cherryPickOnto for why. if len(commit.ParentHashes) > 0 && !shallow[commit.Hash] { parentCommit, pErr := repo.CommitObject(commit.ParentHashes[0]) if pErr != nil { From 8a236eba8939a169796b12579ee502d7c0d5d188 Mon Sep 17 00:00:00 2001 From: Stefan Haubold Date: Fri, 22 May 2026 18:39:27 +0200 Subject: [PATCH 10/12] Scope shallow handling: opt-in via FetchOptions.{Shallow,Unshallow} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two issues raised in review: 1. Generic Fetch auto-unshallowed any shallow repo, including FetchAndCheckoutRemoteBranch's user-branch fetches. A deliberately shallow user clone would silently convert to full history just by running resume/checkout — far outside the metadata-repair scope this PR intended. 2. FetchMetadataTreeOnly / FetchV2MainTreeOnly were no longer "tree- only" after --depth=1 was dropped. They now downloaded the full commit chain of the metadata ref (minus blobs when filtered fetches are enabled). On long-lived checkpoint branches every resume/explain paid the full-history cost. Fix: replace the implicit auto-unshallow with two explicit flags on FetchOptions: - Shallow (--depth=1): set on tip-only probes. Cheap, creates .git/shallow state. Safe now because the reconcile path's commit walk respects .git/shallow (earlier commit in this PR). - Unshallow (--unshallow when shallow): set on metadata-repair paths that need full ancestry (FetchMetadataBranch, FetchV2MainRef, fetchAndRebaseSessionsCommon, doctor v2 disconnection check). Migrates legacy shallow state on demand. Generic FetchAndCheckoutRemoteBranch sets neither — user branches stay as the user configured them. resume.go's getMetadataTree / getV2MetadataTree can revert to always-call-tree-only since tree-only is genuinely cheap again; drops the filtered_fetches-conditional firstFetch switcheroo and its accompanying logging. Test renamed to TestFetch_Unshallow with two cases (Unshallow=true deepens, Unshallow=false leaves alone) plus TestFetch_Shallow asserting --depth=1 actually shallows a clone. Entire-Checkpoint: 9fa8691d4b8a --- cmd/entire/cli/checkpoint/remote/git.go | 33 ++++--- cmd/entire/cli/checkpoint/remote/git_test.go | 87 +++++++++++++------ cmd/entire/cli/git_operations.go | 67 +++++++------- cmd/entire/cli/resume.go | 36 ++------ cmd/entire/cli/strategy/metadata_reconcile.go | 9 +- cmd/entire/cli/strategy/push_common.go | 13 +-- 6 files changed, 137 insertions(+), 108 deletions(-) diff --git a/cmd/entire/cli/checkpoint/remote/git.go b/cmd/entire/cli/checkpoint/remote/git.go index 28281a875d..8054f656be 100644 --- a/cmd/entire/cli/checkpoint/remote/git.go +++ b/cmd/entire/cli/checkpoint/remote/git.go @@ -29,21 +29,28 @@ var sshTokenWarningOnce sync.Once //nolint:gochecknoglobals // intentional per-p // FetchOptions configures a git fetch operation. type FetchOptions struct { - Remote string // remote name or URL (required) - RefSpecs []string // one or more refspecs / object hashes - NoTags bool // adds --no-tags - NoFilter bool // when true, skips --filter=blob:none even if filtered fetches are enabled + Remote string // remote name or URL (required) + RefSpecs []string // one or more refspecs / object hashes + NoTags bool // adds --no-tags + NoFilter bool // when true, skips --filter=blob:none even if filtered fetches are enabled + // Shallow adds --depth=1 to fetch only the tip commit and its tree. Use + // for tip-only probes (e.g. resolving the latest checkpoint metadata) + // where ancestry isn't needed. Creates .git/shallow state — callers that + // later require full history should opt into Unshallow on a follow-up + // fetch. + Shallow bool + // Unshallow adds --unshallow when the repository is currently shallow, + // triggering git to download the rest of the history for the fetched ref. + // Set this on metadata-repair / reconcile paths that need complete + // checkpoint ancestry. Do not set on generic branch fetches — it would + // silently convert a deliberately-shallow user clone into a full one. + Unshallow bool Dir string // working directory (empty = CWD) ExtraArgs []string // additional flags before remote (e.g., "--no-write-fetch-head") } // Fetch runs git fetch with checkpoint token injection and optional // filtered fetches (--filter=blob:none when settings enable it). -// When the repository is shallow, Fetch also adds --unshallow. This migrates -// users away from any shallow checkpoint history (whether produced by an old -// CLI --depth=1 fetch or by an unrelated tool) because the reconcile and -// rebase paths walk parent chains via go-git, which ignores .git/shallow and -// would otherwise traverse stale objects past the shallow boundary. // GIT_TERMINAL_PROMPT=0 is always set. // // Callers that pass a remote name (e.g., "origin") and want filtered fetches to @@ -55,10 +62,10 @@ func Fetch(ctx context.Context, opts FetchOptions) ([]byte, error) { args = append(args, "--no-tags") } args = append(args, opts.ExtraArgs...) - if isShallowRepository(ctx, opts.Dir) { - // Unshallow whenever the repo is shallow so commands that rely on - // checkpoint ancestry/history can operate correctly, regardless of - // how the shallow state was introduced. + switch { + case opts.Shallow: + args = append(args, "--depth=1") + case opts.Unshallow && isShallowRepository(ctx, opts.Dir): args = append(args, "--unshallow") } if !opts.NoFilter && settings.IsFilteredFetchesEnabled(ctx) { diff --git a/cmd/entire/cli/checkpoint/remote/git_test.go b/cmd/entire/cli/checkpoint/remote/git_test.go index 061672389f..c527dcdfcc 100644 --- a/cmd/entire/cli/checkpoint/remote/git_test.go +++ b/cmd/entire/cli/checkpoint/remote/git_test.go @@ -257,36 +257,71 @@ func TestResolveFetchTarget(t *testing.T) { }) } -func TestFetch_UnshallowsShallowRepository(t *testing.T) { +func TestFetch_Unshallow(t *testing.T) { t.Parallel() - for _, tc := range []struct { - name string - settingsContents string - }{ - {"filtered_fetches enabled", `{"enabled": true, "strategy_options": {"filtered_fetches": true}}`}, - {"filtered_fetches disabled", `{"enabled": true}`}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - ctx := context.Background() - - bareDir, cloneDir := setupShallowClone(ctx, t) - testutil.WriteFile(t, cloneDir, ".entire/settings.json", tc.settingsContents) - require.True(t, isShallowRepository(ctx, cloneDir), "test setup should produce a shallow repo") - - _, err := Fetch(ctx, FetchOptions{ - Remote: "file://" + bareDir, - RefSpecs: []string{"+refs/heads/main:refs/remotes/origin/main"}, - NoTags: true, - Dir: cloneDir, - }) - require.NoError(t, err) + t.Run("Unshallow=true deepens a shallow repo", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + bareDir, cloneDir := setupShallowClone(ctx, t) + require.True(t, isShallowRepository(ctx, cloneDir), "test setup should produce a shallow repo") - assert.False(t, isShallowRepository(ctx, cloneDir), - "fetch should unshallow any shallow repository, regardless of filtered_fetches") + out, err := Fetch(ctx, FetchOptions{ + Remote: "file://" + bareDir, + RefSpecs: []string{"+refs/heads/main:refs/remotes/origin/main"}, + NoTags: true, + Unshallow: true, + Dir: cloneDir, }) - } + require.NoError(t, err, "fetch output: %s", out) + + assert.False(t, isShallowRepository(ctx, cloneDir), + "Unshallow=true should remove shallow state when the repo is shallow") + }) + + t.Run("Unshallow=false leaves shallow state alone", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + bareDir, cloneDir := setupShallowClone(ctx, t) + require.True(t, isShallowRepository(ctx, cloneDir)) + + out, err := Fetch(ctx, FetchOptions{ + Remote: "file://" + bareDir, + RefSpecs: []string{"+refs/heads/main:refs/remotes/origin/main"}, + NoTags: true, + Dir: cloneDir, + }) + require.NoError(t, err, "fetch output: %s", out) + + assert.True(t, isShallowRepository(ctx, cloneDir), + "a fetch without Unshallow must not silently convert a shallow repo to a full one") + }) +} + +func TestFetch_Shallow(t *testing.T) { + t.Parallel() + ctx := context.Background() + + bareDir, _ := setupShallowClone(ctx, t) + // Make a fresh non-shallow clone, then fetch with Shallow=true and check + // .git/shallow appears. + cloneDir := t.TempDir() + runIsolatedGit(ctx, t, "", "clone", "--branch", "main", "file://"+bareDir, cloneDir) + require.False(t, isShallowRepository(ctx, cloneDir), "fresh clone should not be shallow") + + out, err := Fetch(ctx, FetchOptions{ + Remote: "file://" + bareDir, + RefSpecs: []string{"+refs/heads/main:refs/remotes/origin/main"}, + NoTags: true, + Shallow: true, + Dir: cloneDir, + }) + require.NoError(t, err, "fetch output: %s", out) + + assert.True(t, isShallowRepository(ctx, cloneDir), + "Shallow=true should request --depth=1 and leave the repo shallow") } // setupShallowClone creates a bare origin, a seed repo with one commit pushed diff --git a/cmd/entire/cli/git_operations.go b/cmd/entire/cli/git_operations.go index 4723fa20c4..03c8ca8328 100644 --- a/cmd/entire/cli/git_operations.go +++ b/cmd/entire/cli/git_operations.go @@ -398,24 +398,28 @@ func FetchAndCheckoutRemoteBranch(ctx context.Context, branchName string) error } // FetchMetadataBranch fetches the entire/checkpoints/v1 branch from origin and -// creates/updates the local branch. The fetch is unfiltered (no --filter=blob:none) -// because callers (resume, explain) need blob content, not just tree structure. +// creates/updates the local branch with full ancestry. Unshallows the repo if +// it was previously left shallow by a tip-only probe, so callers that need +// history (reconcile, rebase) can operate on a complete chain. func FetchMetadataBranch(ctx context.Context) error { - return fetchMetadataFromOrigin(ctx, true /* noFilter */) + return fetchMetadataFromOrigin(ctx, fetchMetadataOpts{NoFilter: true, Unshallow: true}) } -// FetchMetadataTreeOnly fetches the entire/checkpoints/v1 branch from origin -// without blobs. After this call, tree navigation via go-git works but blob -// reads will fail for objects that weren't previously fetched. +// FetchMetadataTreeOnly fetches just the tip of the entire/checkpoints/v1 +// branch (--depth=1). Used by resume/explain to resolve the latest checkpoint +// cheaply without pulling the entire history. May leave .git/shallow set; +// FetchMetadataBranch will undo that when full ancestry is later needed. func FetchMetadataTreeOnly(ctx context.Context) error { - return fetchMetadataFromOrigin(ctx, false /* noFilter */) + return fetchMetadataFromOrigin(ctx, fetchMetadataOpts{Shallow: true}) } -// fetchMetadataFromOrigin fetches the v1 metadata branch from origin into the -// remote-tracking ref refs/remotes/origin/, then safely advances the -// local branch to match. When noFilter is true, --filter=blob:none is suppressed -// so blob content is included. -func fetchMetadataFromOrigin(ctx context.Context, noFilter bool) error { +type fetchMetadataOpts struct { + NoFilter bool + Shallow bool + Unshallow bool +} + +func fetchMetadataFromOrigin(ctx context.Context, fopts fetchMetadataOpts) error { branchName := paths.MetadataBranchName ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) @@ -429,10 +433,12 @@ func fetchMetadataFromOrigin(ctx context.Context, noFilter bool) error { refSpec := fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", branchName, branchName) output, fetchErr := remote.Fetch(ctx, remote.FetchOptions{ - Remote: fetchTarget, - RefSpecs: []string{refSpec}, - NoTags: true, - NoFilter: noFilter, + Remote: fetchTarget, + RefSpecs: []string{refSpec}, + NoTags: true, + NoFilter: fopts.NoFilter, + Shallow: fopts.Shallow, + Unshallow: fopts.Unshallow, }) if fetchErr != nil { if ctx.Err() == context.DeadlineExceeded { @@ -456,23 +462,20 @@ func fetchMetadataFromOrigin(ctx context.Context, noFilter bool) error { return nil } -// FetchV2MainTreeOnly fetches the v2 /main ref from origin without blobs. -// Uses explicit refspec since v2 refs are under refs/entire/, not refs/heads/. +// FetchV2MainTreeOnly fetches just the tip of the v2 /main ref (--depth=1). +// Cheap probe used by resume/explain; may leave .git/shallow set. func FetchV2MainTreeOnly(ctx context.Context) error { - return fetchV2MainFromOrigin(ctx, false /* noFilter */) + return fetchV2MainFromOrigin(ctx, fetchMetadataOpts{Shallow: true}) } -// FetchV2MainRef fetches the v2 /main ref from origin with full blob content. -// The fetch is unfiltered so resume/explain can read metadata JSON blobs. -// Uses explicit refspec since v2 refs are under refs/entire/, not refs/heads/. +// FetchV2MainRef fetches the v2 /main ref from origin with full blob content +// and unshallows the repo if it was previously left shallow by a tip-only +// probe. Used by paths that need complete ancestry (reconcile, rebase). func FetchV2MainRef(ctx context.Context) error { - return fetchV2MainFromOrigin(ctx, true /* noFilter */) + return fetchV2MainFromOrigin(ctx, fetchMetadataOpts{NoFilter: true, Unshallow: true}) } -// fetchV2MainFromOrigin fetches the v2 /main ref from origin into the shared -// staging ref, then promotes it via strategy.PromoteTmpRefSafely. When noFilter -// is true, --filter=blob:none is suppressed. -func fetchV2MainFromOrigin(ctx context.Context, noFilter bool) error { +func fetchV2MainFromOrigin(ctx context.Context, fopts fetchMetadataOpts) error { ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() @@ -484,10 +487,12 @@ func fetchV2MainFromOrigin(ctx context.Context, noFilter bool) error { refSpec := fmt.Sprintf("+%s:%s", paths.V2MainRefName, strategy.V2MainFetchTmpRef) output, fetchErr := remote.Fetch(ctx, remote.FetchOptions{ - Remote: fetchTarget, - RefSpecs: []string{refSpec}, - NoTags: true, - NoFilter: noFilter, + Remote: fetchTarget, + RefSpecs: []string{refSpec}, + NoTags: true, + NoFilter: fopts.NoFilter, + Shallow: fopts.Shallow, + Unshallow: fopts.Unshallow, }) if fetchErr != nil { if ctx.Err() == context.DeadlineExceeded { diff --git a/cmd/entire/cli/resume.go b/cmd/entire/cli/resume.go index b5c4328df4..c6cfd58c3e 100644 --- a/cmd/entire/cli/resume.go +++ b/cmd/entire/cli/resume.go @@ -456,35 +456,26 @@ func getMetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) ) } - // Fetch first so the local lookup below doesn't return stale data when a - // collaborator pushed new checkpoints since our last fetch. Tree-only when - // filtered_fetches is on (cheaper); full fetch otherwise (tree-only would - // degenerate into a full fetch anyway). - firstFetch, firstFetchLabel := FetchMetadataTreeOnly, "treeless-fetch" - if !settings.IsFilteredFetchesEnabled(ctx) { - firstFetch, firstFetchLabel = FetchMetadataBranch, "full-fetch" - } - if fetchErr := firstFetch(ctx); fetchErr == nil { + // Tip-only fetch (--depth=1) is cheap and always runs so the local lookup + // below doesn't return stale data. + if fetchErr := FetchMetadataTreeOnly(ctx); fetchErr == nil { freshRepo, repoErr := openRepository(ctx) if repoErr == nil { - logRefHash(freshRepo, firstFetchLabel) + logRefHash(freshRepo, "treeless-fetch") metadataTree, treeErr := strategy.GetMetadataBranchTree(freshRepo) if treeErr == nil { - logging.Debug(logCtx, "metadata tree obtained via first-attempt fetch", + logging.Debug(logCtx, "metadata tree obtained via treeless fetch", slog.String("tree_hash", metadataTree.Hash.String()), - slog.String("label", firstFetchLabel), ) return metadataTree, freshRepo, nil } - logging.Debug(logCtx, "first-attempt fetch succeeded but tree read failed", + logging.Debug(logCtx, "treeless fetch succeeded but tree read failed", slog.String("error", treeErr.Error()), - slog.String("label", firstFetchLabel), ) } } else { - logging.Debug(logCtx, "first-attempt fetch failed, trying local", + logging.Debug(logCtx, "treeless fetch failed, trying local", slog.String("error", fetchErr.Error()), - slog.String("label", firstFetchLabel), ) } @@ -547,18 +538,7 @@ func getMetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) // getV2MetadataTree resolves the v2 /main ref tree with the same // fetch fallback pattern as getMetadataTree, including checkpoint remote support. func getV2MetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) { - // Fetch first so the local lookup doesn't return stale data. When - // filtered_fetches is off, use the full fetch directly (tree-only would - // degenerate into the same call) and pass nil as the fallback to avoid a - // duplicate round-trip. - firstFetch := checkpoint.FetchRefFunc(FetchV2MainTreeOnly) - secondFetch := checkpoint.FetchRefFunc(FetchV2MainRef) - if !settings.IsFilteredFetchesEnabled(ctx) { - firstFetch = FetchV2MainRef - secondFetch = nil - } - - tree, repo, err := checkpoint.GetV2MetadataTree(ctx, firstFetch, secondFetch, openRepository) + tree, repo, err := checkpoint.GetV2MetadataTree(ctx, FetchV2MainTreeOnly, FetchV2MainRef, openRepository) if err == nil { return tree, repo, nil } diff --git a/cmd/entire/cli/strategy/metadata_reconcile.go b/cmd/entire/cli/strategy/metadata_reconcile.go index fc01013371..7007cfc21f 100644 --- a/cmd/entire/cli/strategy/metadata_reconcile.go +++ b/cmd/entire/cli/strategy/metadata_reconcile.go @@ -411,10 +411,11 @@ func fetchRefToTemp(ctx context.Context, repoPath, remoteName, srcRef, dstRef st refspec := fmt.Sprintf("+%s:%s", srcRef, dstRef) output, err := remote.Fetch(ctx, remote.FetchOptions{ - Remote: fetchTarget, - RefSpecs: []string{refspec}, - NoTags: true, - Dir: repoPath, + Remote: fetchTarget, + RefSpecs: []string{refspec}, + NoTags: true, + Unshallow: true, + Dir: repoPath, }) if err != nil { redactedURL := remote.RedactURL(fetchTarget) diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index 594219f1a6..63b56b1a84 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -327,13 +327,14 @@ func fetchAndRebaseSessionsCommon(ctx context.Context, target, branchName string } // Use git CLI for fetch (go-git's fetch can be tricky with auth). - // Use --filter=blob:none for a partial fetch that downloads only commits - // and trees, skipping blobs. The merge only needs the tree structure to - // combine entries; blobs are already local or fetched on demand. + // Unshallow when needed so merge-base and ancestry walks see the real + // shared history rather than the local shallow view, which would + // otherwise misread a healthy chain as "disconnected". if output, fetchErr := remote.Fetch(ctx, remote.FetchOptions{ - Remote: fetchTarget, - RefSpecs: []string{refSpec}, - NoTags: true, + Remote: fetchTarget, + RefSpecs: []string{refSpec}, + NoTags: true, + Unshallow: true, }); fetchErr != nil { return fmt.Errorf("fetch failed: %s", output) } From e639b76c8d9a26a2cd9bdae45f3d87532f8a9569 Mon Sep 17 00:00:00 2001 From: Stefan Haubold Date: Fri, 22 May 2026 19:41:31 +0200 Subject: [PATCH 11/12] Don't --unshallow on push hot path: it's a global op, not ref-scoped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git's --unshallow removes ALL shallow boundaries in .git/shallow, not just the one for the ref being fetched. On a shallow clone of a large repo it pulls the entire repository history (we observed 687 MB on entirehq/entire). Used from the push hook, this hangs every push on shallow repos until the full clone arrives. The downstream reconcile/rebase paths already respect .git/shallow (collectCommitChain stops at shallow boundaries; collectCommitsSince uses git rev-list which respects them natively). When the local view is shallow-truncated, the merge-base check may report "disconnected" where the actual chains share ancestry below the boundary — but the cherry-pick path it routes through still produces a correct result, because it picks up exactly the commits that aren't yet on the remote. Remove Unshallow from: - fetchAndRebaseSessionsCommon (push hot path) - FetchMetadataBranch (resume/explain fallback — only needs blobs) - FetchV2MainRef (same) Keep Unshallow on metadata_reconcile's doctor v2 disconnection check since it's only reached from `entire doctor`, an explicit user-invoked diagnostic where the cost is acceptable. Verified by re-running the full unit suite (5304 tests) and the fetch-flag regression coverage from TestFetch_Unshallow / TestFetch_Shallow. Entire-Checkpoint: 8184f5f2a970 --- cmd/entire/cli/git_operations.go | 18 +++++++++--------- cmd/entire/cli/strategy/push_common.go | 16 +++++++++------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/cmd/entire/cli/git_operations.go b/cmd/entire/cli/git_operations.go index 03c8ca8328..83395c7402 100644 --- a/cmd/entire/cli/git_operations.go +++ b/cmd/entire/cli/git_operations.go @@ -397,12 +397,13 @@ func FetchAndCheckoutRemoteBranch(ctx context.Context, branchName string) error return CheckoutBranch(ctx, branchName) } -// FetchMetadataBranch fetches the entire/checkpoints/v1 branch from origin and -// creates/updates the local branch with full ancestry. Unshallows the repo if -// it was previously left shallow by a tip-only probe, so callers that need -// history (reconcile, rebase) can operate on a complete chain. +// FetchMetadataBranch fetches the entire/checkpoints/v1 branch from origin +// with full blob content. Used as a fallback by resume/explain when the +// tree-only probe is insufficient (e.g. the metadata.json blob is missing). +// Does NOT --unshallow: --unshallow is a global property of the clone, so on +// shallow checkpoint repos it would also deepen unrelated branches. func FetchMetadataBranch(ctx context.Context) error { - return fetchMetadataFromOrigin(ctx, fetchMetadataOpts{NoFilter: true, Unshallow: true}) + return fetchMetadataFromOrigin(ctx, fetchMetadataOpts{NoFilter: true}) } // FetchMetadataTreeOnly fetches just the tip of the entire/checkpoints/v1 @@ -468,11 +469,10 @@ func FetchV2MainTreeOnly(ctx context.Context) error { return fetchV2MainFromOrigin(ctx, fetchMetadataOpts{Shallow: true}) } -// FetchV2MainRef fetches the v2 /main ref from origin with full blob content -// and unshallows the repo if it was previously left shallow by a tip-only -// probe. Used by paths that need complete ancestry (reconcile, rebase). +// FetchV2MainRef fetches the v2 /main ref from origin with full blob content. +// Does NOT --unshallow: see FetchMetadataBranch for the reasoning. func FetchV2MainRef(ctx context.Context) error { - return fetchV2MainFromOrigin(ctx, fetchMetadataOpts{NoFilter: true, Unshallow: true}) + return fetchV2MainFromOrigin(ctx, fetchMetadataOpts{NoFilter: true}) } func fetchV2MainFromOrigin(ctx context.Context, fopts fetchMetadataOpts) error { diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index 63b56b1a84..8dd8c8b612 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -327,14 +327,16 @@ func fetchAndRebaseSessionsCommon(ctx context.Context, target, branchName string } // Use git CLI for fetch (go-git's fetch can be tricky with auth). - // Unshallow when needed so merge-base and ancestry walks see the real - // shared history rather than the local shallow view, which would - // otherwise misread a healthy chain as "disconnected". + // Do NOT --unshallow here: on a shallow repo with deep history (e.g. a + // shared monorepo), --unshallow downloads the whole repository because + // git treats shallow as a global property of the clone, not per-ref. + // The downstream reconcile/rebase paths walk only commits visible past + // .git/shallow (collectCommitChain / collectCommitsSince), so the + // missing pre-shallow history isn't needed to produce a correct rebase. if output, fetchErr := remote.Fetch(ctx, remote.FetchOptions{ - Remote: fetchTarget, - RefSpecs: []string{refSpec}, - NoTags: true, - Unshallow: true, + Remote: fetchTarget, + RefSpecs: []string{refSpec}, + NoTags: true, }); fetchErr != nil { return fmt.Errorf("fetch failed: %s", output) } From eccac05c420224dbc70b94cc2a07391a7ae6ec86 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Wed, 27 May 2026 13:34:58 +0100 Subject: [PATCH 12/12] Fix tests Signed-off-by: Paulo Gomes Entire-Checkpoint: 1fe71f299748 --- .../cli/fetch_no_config_pollution_test.go | 55 +++++++++++++++++++ cmd/entire/cli/git_operations.go | 11 +++- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/cmd/entire/cli/fetch_no_config_pollution_test.go b/cmd/entire/cli/fetch_no_config_pollution_test.go index f90d1dc2fa..2639e9f18d 100644 --- a/cmd/entire/cli/fetch_no_config_pollution_test.go +++ b/cmd/entire/cli/fetch_no_config_pollution_test.go @@ -8,8 +8,13 @@ import ( "strings" "testing" + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" "github.com/entireio/cli/cmd/entire/cli/paths" "github.com/entireio/cli/cmd/entire/cli/testutil" + "github.com/entireio/cli/redact" + "github.com/go-git/go-git/v6" + "github.com/stretchr/testify/require" ) // TestFetchDoesNotPolluteOriginConfig is a regression test for #712. @@ -94,6 +99,56 @@ func TestFetchDoesNotPolluteOriginConfig(t *testing.T) { } } +// TestFetchV2MainTreeOnly_DoesNotCreateShallowRepository guards the explain +// remote-fetch path for stale local v2 refs. V2 fetches promote through +// SafelyAdvanceLocalRef, which needs ancestry to prove a fast-forward. If this +// helper creates a shallow boundary, a remote descendant can look diverged and +// the local v2 ref stays stale. +func TestFetchV2MainTreeOnly_DoesNotCreateShallowRepository(t *testing.T) { + // Uses t.Chdir() — cannot run in parallel. + + tmpDir := t.TempDir() + bareDir := filepath.Join(tmpDir, "bare.git") + producerDir := filepath.Join(tmpDir, "producer") + localDir := filepath.Join(tmpDir, "local") + + runGit(t, tmpDir, "init", "--bare", bareDir) + + testutil.InitRepo(t, producerDir) + testutil.WriteFile(t, producerDir, "README.md", "hello") + testutil.GitAdd(t, producerDir, "README.md") + testutil.GitCommit(t, producerDir, "init") + runGit(t, producerDir, "remote", "add", "origin", bareDir) + + producerRepo, err := git.PlainOpen(producerDir) + if err != nil { + t.Fatalf("failed to open producer repo: %v", err) + } + writeV2CheckpointForExport(t, producerRepo, id.MustCheckpointID("121212121212"), checkpoint.WriteCommittedOptions{ + SessionID: "fetch-v2-shallow-guard", + Transcript: redact.AlreadyRedacted([]byte(`{"type":"user","message":{"content":[{"type":"text","text":"hello"}]}}` + "\n")), + }) + + runGit(t, producerDir, "push", "origin", "HEAD:refs/heads/main", paths.V2MainRefName+":"+paths.V2MainRefName) + runGit(t, bareDir, "symbolic-ref", "HEAD", "refs/heads/main") + runGit(t, tmpDir, "clone", "--branch", "main", bareDir, localDir) + + require.NoError(t, os.MkdirAll(filepath.Join(localDir, ".entire"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(localDir, ".entire", "settings.json"), + []byte(`{"enabled": true, "strategy_options": {"filtered_fetches": true}}`), + 0o644, + )) + + t.Chdir(localDir) + + require.NoError(t, FetchV2MainTreeOnly(context.Background())) + + if got := gitOutput(t, localDir, "rev-parse", "--is-shallow-repository"); got != "false" { + t.Fatalf("FetchV2MainTreeOnly left repository shallow = %s, want false", got) + } +} + func runGit(t *testing.T, dir string, args ...string) { t.Helper() cmd := exec.CommandContext(t.Context(), "git", args...) diff --git a/cmd/entire/cli/git_operations.go b/cmd/entire/cli/git_operations.go index 83395c7402..a1d67434f3 100644 --- a/cmd/entire/cli/git_operations.go +++ b/cmd/entire/cli/git_operations.go @@ -463,10 +463,15 @@ func fetchMetadataFromOrigin(ctx context.Context, fopts fetchMetadataOpts) error return nil } -// FetchV2MainTreeOnly fetches just the tip of the v2 /main ref (--depth=1). -// Cheap probe used by resume/explain; may leave .git/shallow set. +// FetchV2MainTreeOnly fetches the v2 /main ref for read-only lookup paths. +// +// Unlike the v1 metadata branch, v2 custom refs do not have a remote-tracking +// fallback tree. Avoid --depth=1 here: a shallow fetch can make a remote +// descendant look unrelated to go-git, causing SafelyAdvanceLocalRef to +// preserve a stale local ref and explain/resume to miss freshly fetched +// checkpoints. func FetchV2MainTreeOnly(ctx context.Context) error { - return fetchV2MainFromOrigin(ctx, fetchMetadataOpts{Shallow: true}) + return fetchV2MainFromOrigin(ctx, fetchMetadataOpts{}) } // FetchV2MainRef fetches the v2 /main ref from origin with full blob content.