diff --git a/cmd/entire/cli/checkpoint/remote/git.go b/cmd/entire/cli/checkpoint/remote/git.go index e24cdc938d..8054f656be 100644 --- a/cmd/entire/cli/checkpoint/remote/git.go +++ b/cmd/entire/cli/checkpoint/remote/git.go @@ -29,11 +29,22 @@ var sshTokenWarningOnce sync.Once //nolint:gochecknoglobals // intentional per-p // FetchOptions configures a git fetch operation. type FetchOptions struct { - Remote string // remote name or URL (required) - RefSpecs []string // one or more refspecs / object hashes - Shallow bool // adds --depth=1 - NoTags bool // adds --no-tags - NoFilter bool // when true, skips --filter=blob:none even if filtered fetches are enabled + Remote string // remote name or URL (required) + RefSpecs []string // one or more refspecs / object hashes + NoTags bool // adds --no-tags + NoFilter bool // when true, skips --filter=blob:none even if filtered fetches are enabled + // Shallow adds --depth=1 to fetch only the tip commit and its tree. Use + // for tip-only probes (e.g. resolving the latest checkpoint metadata) + // where ancestry isn't needed. Creates .git/shallow state — callers that + // later require full history should opt into Unshallow on a follow-up + // fetch. + Shallow bool + // Unshallow adds --unshallow when the repository is currently shallow, + // triggering git to download the rest of the history for the fetched ref. + // Set this on metadata-repair / reconcile paths that need complete + // checkpoint ancestry. Do not set on generic branch fetches — it would + // silently convert a deliberately-shallow user clone into a full one. + Unshallow bool Dir string // working directory (empty = CWD) ExtraArgs []string // additional flags before remote (e.g., "--no-write-fetch-head") } @@ -46,14 +57,17 @@ type FetchOptions struct { // resolve the name to a URL (to avoid persisting promisor settings) should call // ResolveFetchTarget first and pass the resolved target as opts.Remote. func Fetch(ctx context.Context, opts FetchOptions) ([]byte, error) { - args := []string{"fetch"} + args := []string{"fetch", "--no-auto-gc"} if opts.NoTags { args = append(args, "--no-tags") } - if opts.Shallow { + args = append(args, opts.ExtraArgs...) + switch { + case opts.Shallow: args = append(args, "--depth=1") + case opts.Unshallow && isShallowRepository(ctx, opts.Dir): + args = append(args, "--unshallow") } - args = append(args, opts.ExtraArgs...) if !opts.NoFilter && settings.IsFilteredFetchesEnabled(ctx) { args = append(args, "--filter=blob:none") } @@ -309,6 +323,20 @@ func ResolveFetchTarget(ctx context.Context, target string) (string, error) { return url, nil } +// isShallowRepository returns true when the git repository at dir is shallow. +// An empty dir inherits the parent process's working directory, matching the +// semantics callers use when invoking Fetch with empty FetchOptions.Dir. +func isShallowRepository(ctx context.Context, dir string) bool { + cmd := exec.CommandContext(ctx, "git", "rev-parse", "--is-shallow-repository") + cmd.Dir = dir + disableTerminalPrompt(cmd) + out, err := cmd.Output() + if err != nil { + return false + } + return strings.TrimSpace(string(out)) == "true" +} + // newCommand creates an exec.Cmd for a git operation that may need // checkpoint token authentication. If ENTIRE_CHECKPOINT_TOKEN is set: // - if the target in args is (or resolves to) an SSH remote, the target is diff --git a/cmd/entire/cli/checkpoint/remote/git_test.go b/cmd/entire/cli/checkpoint/remote/git_test.go index a6a9958224..c527dcdfcc 100644 --- a/cmd/entire/cli/checkpoint/remote/git_test.go +++ b/cmd/entire/cli/checkpoint/remote/git_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "os" "os/exec" + "path/filepath" "strings" "sync" "testing" @@ -27,10 +28,10 @@ func TestExtractRemoteFromArgs(t *testing.T) { args []string want string }{ - {"fetch with URL", []string{"fetch", "https://github.com/org/repo.git", "refs/heads/main"}, "https://github.com/org/repo.git"}, + {"fetch with URL", []string{"fetch", "--no-auto-gc", "https://github.com/org/repo.git", "refs/heads/main"}, "https://github.com/org/repo.git"}, {"push with flags", []string{"push", "--no-verify", "--porcelain", "origin", "main"}, "origin"}, {"ls-remote", []string{"ls-remote", "origin", "refs/heads/*"}, "origin"}, - {"fetch with filter", []string{"fetch", "--no-tags", "--filter=blob:none", "https://host/r.git", "+refs/heads/main:refs/tmp"}, "https://host/r.git"}, + {"fetch with filter", []string{"fetch", "--no-auto-gc", "--no-tags", "--filter=blob:none", "https://host/r.git", "+refs/heads/main:refs/tmp"}, "https://host/r.git"}, {"empty args", []string{}, ""}, {"subcommand only", []string{"fetch"}, ""}, {"only flags", []string{"fetch", "--no-tags"}, ""}, @@ -256,6 +257,112 @@ func TestResolveFetchTarget(t *testing.T) { }) } +func TestFetch_Unshallow(t *testing.T) { + t.Parallel() + + t.Run("Unshallow=true deepens a shallow repo", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + bareDir, cloneDir := setupShallowClone(ctx, t) + require.True(t, isShallowRepository(ctx, cloneDir), "test setup should produce a shallow repo") + + out, err := Fetch(ctx, FetchOptions{ + Remote: "file://" + bareDir, + RefSpecs: []string{"+refs/heads/main:refs/remotes/origin/main"}, + NoTags: true, + Unshallow: true, + Dir: cloneDir, + }) + require.NoError(t, err, "fetch output: %s", out) + + assert.False(t, isShallowRepository(ctx, cloneDir), + "Unshallow=true should remove shallow state when the repo is shallow") + }) + + t.Run("Unshallow=false leaves shallow state alone", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + bareDir, cloneDir := setupShallowClone(ctx, t) + require.True(t, isShallowRepository(ctx, cloneDir)) + + out, err := Fetch(ctx, FetchOptions{ + Remote: "file://" + bareDir, + RefSpecs: []string{"+refs/heads/main:refs/remotes/origin/main"}, + NoTags: true, + Dir: cloneDir, + }) + require.NoError(t, err, "fetch output: %s", out) + + assert.True(t, isShallowRepository(ctx, cloneDir), + "a fetch without Unshallow must not silently convert a shallow repo to a full one") + }) +} + +func TestFetch_Shallow(t *testing.T) { + t.Parallel() + ctx := context.Background() + + bareDir, _ := setupShallowClone(ctx, t) + // Make a fresh non-shallow clone, then fetch with Shallow=true and check + // .git/shallow appears. + cloneDir := t.TempDir() + runIsolatedGit(ctx, t, "", "clone", "--branch", "main", "file://"+bareDir, cloneDir) + require.False(t, isShallowRepository(ctx, cloneDir), "fresh clone should not be shallow") + + out, err := Fetch(ctx, FetchOptions{ + Remote: "file://" + bareDir, + RefSpecs: []string{"+refs/heads/main:refs/remotes/origin/main"}, + NoTags: true, + Shallow: true, + Dir: cloneDir, + }) + require.NoError(t, err, "fetch output: %s", out) + + assert.True(t, isShallowRepository(ctx, cloneDir), + "Shallow=true should request --depth=1 and leave the repo shallow") +} + +// setupShallowClone creates a bare origin, a seed repo with one commit pushed +// to it, a shallow (--depth=1) clone, and then advances origin by one more +// commit so that a subsequent fetch into the clone has work to do. Returns the +// bare origin path and the shallow clone path. +func setupShallowClone(ctx context.Context, t *testing.T) (bareDir, cloneDir string) { + t.Helper() + tmpDir := t.TempDir() + bareDir = filepath.Join(tmpDir, "bare.git") + seedDir := filepath.Join(tmpDir, "seed") + cloneDir = filepath.Join(tmpDir, "clone") + + testutil.InitRepo(t, seedDir) + testutil.WriteFile(t, seedDir, "f.txt", "init") + testutil.GitAdd(t, seedDir, "f.txt") + testutil.GitCommit(t, seedDir, "init") + + runIsolatedGit(ctx, t, "", "init", "--bare", bareDir) + runIsolatedGit(ctx, t, seedDir, "remote", "add", "origin", bareDir) + runIsolatedGit(ctx, t, seedDir, "push", "origin", "HEAD:refs/heads/main") + runIsolatedGit(ctx, t, "", "clone", "--depth=1", "--branch", "main", "file://"+bareDir, cloneDir) + + testutil.WriteFile(t, seedDir, "f.txt", "init\nnext\n") + testutil.GitAdd(t, seedDir, "f.txt") + testutil.GitCommit(t, seedDir, "next") + runIsolatedGit(ctx, t, seedDir, "push", "origin", "HEAD:refs/heads/main") + + return bareDir, cloneDir +} + +func runIsolatedGit(ctx context.Context, t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.CommandContext(ctx, "git", args...) + if dir != "" { + cmd.Dir = dir + } + cmd.Env = testutil.GitIsolatedEnv() + require.NoError(t, cmd.Run(), "git %v", args) +} + func TestAppendCheckpointTokenEnv(t *testing.T) { t.Parallel() @@ -591,7 +698,7 @@ func TestNewCommand_GIT_TERMINAL_PROMPT_Coexistence(t *testing.T) { t.Setenv(CheckpointTokenEnvVar, "coexist-token") cmd := newCommand(context.Background(), - "fetch", "--no-tags", "--filter=blob:none", "https://github.com/org/repo.git", "refs/heads/main") + "fetch", "--no-auto-gc", "--no-tags", "--filter=blob:none", "https://github.com/org/repo.git", "refs/heads/main") require.NotNil(t, cmd.Env) cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0") diff --git a/cmd/entire/cli/fetch_no_config_pollution_test.go b/cmd/entire/cli/fetch_no_config_pollution_test.go index f90d1dc2fa..2639e9f18d 100644 --- a/cmd/entire/cli/fetch_no_config_pollution_test.go +++ b/cmd/entire/cli/fetch_no_config_pollution_test.go @@ -8,8 +8,13 @@ import ( "strings" "testing" + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" "github.com/entireio/cli/cmd/entire/cli/paths" "github.com/entireio/cli/cmd/entire/cli/testutil" + "github.com/entireio/cli/redact" + "github.com/go-git/go-git/v6" + "github.com/stretchr/testify/require" ) // TestFetchDoesNotPolluteOriginConfig is a regression test for #712. @@ -94,6 +99,56 @@ func TestFetchDoesNotPolluteOriginConfig(t *testing.T) { } } +// TestFetchV2MainTreeOnly_DoesNotCreateShallowRepository guards the explain +// remote-fetch path for stale local v2 refs. V2 fetches promote through +// SafelyAdvanceLocalRef, which needs ancestry to prove a fast-forward. If this +// helper creates a shallow boundary, a remote descendant can look diverged and +// the local v2 ref stays stale. +func TestFetchV2MainTreeOnly_DoesNotCreateShallowRepository(t *testing.T) { + // Uses t.Chdir() — cannot run in parallel. + + tmpDir := t.TempDir() + bareDir := filepath.Join(tmpDir, "bare.git") + producerDir := filepath.Join(tmpDir, "producer") + localDir := filepath.Join(tmpDir, "local") + + runGit(t, tmpDir, "init", "--bare", bareDir) + + testutil.InitRepo(t, producerDir) + testutil.WriteFile(t, producerDir, "README.md", "hello") + testutil.GitAdd(t, producerDir, "README.md") + testutil.GitCommit(t, producerDir, "init") + runGit(t, producerDir, "remote", "add", "origin", bareDir) + + producerRepo, err := git.PlainOpen(producerDir) + if err != nil { + t.Fatalf("failed to open producer repo: %v", err) + } + writeV2CheckpointForExport(t, producerRepo, id.MustCheckpointID("121212121212"), checkpoint.WriteCommittedOptions{ + SessionID: "fetch-v2-shallow-guard", + Transcript: redact.AlreadyRedacted([]byte(`{"type":"user","message":{"content":[{"type":"text","text":"hello"}]}}` + "\n")), + }) + + runGit(t, producerDir, "push", "origin", "HEAD:refs/heads/main", paths.V2MainRefName+":"+paths.V2MainRefName) + runGit(t, bareDir, "symbolic-ref", "HEAD", "refs/heads/main") + runGit(t, tmpDir, "clone", "--branch", "main", bareDir, localDir) + + require.NoError(t, os.MkdirAll(filepath.Join(localDir, ".entire"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(localDir, ".entire", "settings.json"), + []byte(`{"enabled": true, "strategy_options": {"filtered_fetches": true}}`), + 0o644, + )) + + t.Chdir(localDir) + + require.NoError(t, FetchV2MainTreeOnly(context.Background())) + + if got := gitOutput(t, localDir, "rev-parse", "--is-shallow-repository"); got != "false" { + t.Fatalf("FetchV2MainTreeOnly left repository shallow = %s, want false", got) + } +} + func runGit(t *testing.T, dir string, args ...string) { t.Helper() cmd := exec.CommandContext(t.Context(), "git", args...) diff --git a/cmd/entire/cli/git_operations.go b/cmd/entire/cli/git_operations.go index 16076d78b3..a1d67434f3 100644 --- a/cmd/entire/cli/git_operations.go +++ b/cmd/entire/cli/git_operations.go @@ -397,27 +397,30 @@ func FetchAndCheckoutRemoteBranch(ctx context.Context, branchName string) error return CheckoutBranch(ctx, branchName) } -// FetchMetadataBranch fetches the entire/checkpoints/v1 branch from origin and -// creates/updates the local branch. The fetch is unfiltered (no --filter=blob:none) -// because callers (resume, explain) need blob content, not just tree structure. +// FetchMetadataBranch fetches the entire/checkpoints/v1 branch from origin +// with full blob content. Used as a fallback by resume/explain when the +// tree-only probe is insufficient (e.g. the metadata.json blob is missing). +// Does NOT --unshallow: --unshallow is a global property of the clone, so on +// shallow checkpoint repos it would also deepen unrelated branches. func FetchMetadataBranch(ctx context.Context) error { - return fetchMetadataFromOrigin(ctx, false /* shallow */, true /* noFilter */) + return fetchMetadataFromOrigin(ctx, fetchMetadataOpts{NoFilter: true}) } -// FetchMetadataTreeOnly fetches the tip of the entire/checkpoints/v1 branch -// from origin with --depth=1, downloading only the latest commit and its tree -// objects. After this call, tree navigation via go-git works but blob reads -// will fail for objects that weren't previously fetched. +// FetchMetadataTreeOnly fetches just the tip of the entire/checkpoints/v1 +// branch (--depth=1). Used by resume/explain to resolve the latest checkpoint +// cheaply without pulling the entire history. May leave .git/shallow set; +// FetchMetadataBranch will undo that when full ancestry is later needed. func FetchMetadataTreeOnly(ctx context.Context) error { - return fetchMetadataFromOrigin(ctx, true /* shallow */, false /* noFilter */) + return fetchMetadataFromOrigin(ctx, fetchMetadataOpts{Shallow: true}) } -// fetchMetadataFromOrigin fetches the v1 metadata branch from origin into the -// remote-tracking ref refs/remotes/origin/, then safely advances the -// local branch to match. When shallow is true, --depth=1 is added so only -// the tip is downloaded. When noFilter is true, --filter=blob:none is suppressed -// so blob content is included. -func fetchMetadataFromOrigin(ctx context.Context, shallow, noFilter bool) error { +type fetchMetadataOpts struct { + NoFilter bool + Shallow bool + Unshallow bool +} + +func fetchMetadataFromOrigin(ctx context.Context, fopts fetchMetadataOpts) error { branchName := paths.MetadataBranchName ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) @@ -431,11 +434,12 @@ func fetchMetadataFromOrigin(ctx context.Context, shallow, noFilter bool) error refSpec := fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", branchName, branchName) output, fetchErr := remote.Fetch(ctx, remote.FetchOptions{ - Remote: fetchTarget, - RefSpecs: []string{refSpec}, - NoTags: true, - Shallow: shallow, - NoFilter: noFilter, + Remote: fetchTarget, + RefSpecs: []string{refSpec}, + NoTags: true, + NoFilter: fopts.NoFilter, + Shallow: fopts.Shallow, + Unshallow: fopts.Unshallow, }) if fetchErr != nil { if ctx.Err() == context.DeadlineExceeded { @@ -459,25 +463,24 @@ func fetchMetadataFromOrigin(ctx context.Context, shallow, noFilter bool) error return nil } -// FetchV2MainTreeOnly fetches the tip of the v2 /main ref from origin with -// --depth=1, downloading only the latest commit and its tree objects. -// Uses explicit refspec since v2 refs are under refs/entire/, not refs/heads/. +// FetchV2MainTreeOnly fetches the v2 /main ref for read-only lookup paths. +// +// Unlike the v1 metadata branch, v2 custom refs do not have a remote-tracking +// fallback tree. Avoid --depth=1 here: a shallow fetch can make a remote +// descendant look unrelated to go-git, causing SafelyAdvanceLocalRef to +// preserve a stale local ref and explain/resume to miss freshly fetched +// checkpoints. func FetchV2MainTreeOnly(ctx context.Context) error { - return fetchV2MainFromOrigin(ctx, true /* shallow */, false /* noFilter */) + return fetchV2MainFromOrigin(ctx, fetchMetadataOpts{}) } // FetchV2MainRef fetches the v2 /main ref from origin with full blob content. -// The fetch is unfiltered so resume/explain can read metadata JSON blobs. -// Uses explicit refspec since v2 refs are under refs/entire/, not refs/heads/. +// Does NOT --unshallow: see FetchMetadataBranch for the reasoning. func FetchV2MainRef(ctx context.Context) error { - return fetchV2MainFromOrigin(ctx, false /* shallow */, true /* noFilter */) + return fetchV2MainFromOrigin(ctx, fetchMetadataOpts{NoFilter: true}) } -// fetchV2MainFromOrigin fetches the v2 /main ref from origin into the shared -// staging ref, then promotes it via strategy.PromoteTmpRefSafely. When -// shallow is true, --depth=1 is added so only the tip is downloaded. -// When noFilter is true, --filter=blob:none is suppressed. -func fetchV2MainFromOrigin(ctx context.Context, shallow, noFilter bool) error { +func fetchV2MainFromOrigin(ctx context.Context, fopts fetchMetadataOpts) error { ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() @@ -489,11 +492,12 @@ func fetchV2MainFromOrigin(ctx context.Context, shallow, noFilter bool) error { refSpec := fmt.Sprintf("+%s:%s", paths.V2MainRefName, strategy.V2MainFetchTmpRef) output, fetchErr := remote.Fetch(ctx, remote.FetchOptions{ - Remote: fetchTarget, - RefSpecs: []string{refSpec}, - NoTags: true, - Shallow: shallow, - NoFilter: noFilter, + Remote: fetchTarget, + RefSpecs: []string{refSpec}, + NoTags: true, + NoFilter: fopts.NoFilter, + Shallow: fopts.Shallow, + Unshallow: fopts.Unshallow, }) if fetchErr != nil { if ctx.Err() == context.DeadlineExceeded { diff --git a/cmd/entire/cli/resume.go b/cmd/entire/cli/resume.go index 129cf9af52..c6cfd58c3e 100644 --- a/cmd/entire/cli/resume.go +++ b/cmd/entire/cli/resume.go @@ -456,7 +456,8 @@ func getMetadataTree(ctx context.Context) (*object.Tree, *git.Repository, error) ) } - // Try treeless fetch from origin + // Tip-only fetch (--depth=1) is cheap and always runs so the local lookup + // below doesn't return stale data. if fetchErr := FetchMetadataTreeOnly(ctx); fetchErr == nil { freshRepo, repoErr := openRepository(ctx) if repoErr == nil { diff --git a/cmd/entire/cli/strategy/common.go b/cmd/entire/cli/strategy/common.go index 104a138427..788a143c35 100644 --- a/cmd/entire/cli/strategy/common.go +++ b/cmd/entire/cli/strategy/common.go @@ -129,22 +129,19 @@ func PromoteTmpRefSafely(ctx context.Context, tmpRefName, destRefName plumbing.R // SafelyAdvanceLocalRef updates localRefName to point at targetHash only when // the move is non-destructive: // -// - local missing → create at target -// - local == target → no-op -// - local at or ahead of target (target is an ancestor of local) → no-op -// - local strictly behind target (local is an ancestor of target) → fast-forward +// - local missing: create at target +// - local == target: no-op +// - local at or ahead of target (target is an ancestor of local): no-op +// - local strictly behind target (local is an ancestor of target): fast-forward // - diverged or unrelated history (neither ref is an ancestor of the other) -// → no-op, logged at debug level +// no-op, logged at debug level // // The diverged case is the dangerous one this guard exists for. The CLI // maintains orphan-style refs (entire/checkpoints/v1, the V2 main ref) where -// unpushed local commits encode user work — a fetch that landed sibling +// unpushed local commits encode user work. A fetch that landed sibling // commits from another machine must not silently rewind that work. In the // diverged case readers fall through to the remote-tracking tree or to v1, // so resume keeps working while local-only commits stay reachable. -// -// The ancestry checks walk from the local ref (which has full history), so -// callers that fetched with --depth=1 do not break the check. func SafelyAdvanceLocalRef(ctx context.Context, repo *git.Repository, localRefName plumbing.ReferenceName, targetHash plumbing.Hash) error { currentLocal, localErr := repo.Reference(localRefName, true) if localErr == nil { diff --git a/cmd/entire/cli/strategy/metadata_reconcile.go b/cmd/entire/cli/strategy/metadata_reconcile.go index ccb616bb44..7007cfc21f 100644 --- a/cmd/entire/cli/strategy/metadata_reconcile.go +++ b/cmd/entire/cli/strategy/metadata_reconcile.go @@ -8,10 +8,13 @@ import ( "log/slog" "os" "os/exec" + "path/filepath" + "strings" "sync" "time" "github.com/entireio/cli/cmd/entire/cli/checkpoint" + remote "github.com/entireio/cli/cmd/entire/cli/checkpoint/remote" "github.com/entireio/cli/cmd/entire/cli/logging" "github.com/entireio/cli/cmd/entire/cli/paths" @@ -151,8 +154,13 @@ func ReconcileDisconnectedMetadataBranch( // Disconnected — cherry-pick local commits onto remote tip fmt.Fprintln(w, "[entire] Detected disconnected session metadata (local and remote share no common ancestor)") + shallow, err := loadShallowHashes(ctx, repoPath) + if err != nil { + return fmt.Errorf("failed to load shallow boundaries: %w", err) + } + // Collect local commits oldest-first - localCommits, err := collectCommitChain(repo, localHash) + localCommits, err := collectCommitChain(repo, localHash, shallow) if err != nil { return fmt.Errorf("failed to collect local commits: %w", err) } @@ -181,7 +189,7 @@ func ReconcileDisconnectedMetadataBranch( fmt.Fprintf(w, "[entire] Cherry-picking %d local checkpoint(s) onto remote...\n", len(dataCommits)) - newTip, err := cherryPickOnto(ctx, repo, remoteHash, dataCommits) + newTip, err := cherryPickOnto(ctx, repo, remoteHash, dataCommits, shallow) if err != nil { return fmt.Errorf("failed to cherry-pick local commits onto remote: %w", err) } @@ -196,6 +204,244 @@ func ReconcileDisconnectedMetadataBranch( return nil } +// v2DoctorTmpRef is the temporary ref used by doctor to fetch and compare the remote v2 /main. +// Uses the refs/entire-fetch-tmp/ namespace consistent with checkpoint_remote.go. +const v2DoctorTmpRef = "refs/entire-fetch-tmp/doctor-v2-main" + +// IsV2MainDisconnected checks whether the local v2 /main ref and the remote +// v2 /main ref exist but share no common ancestor. Uses git ls-remote to +// discover the remote ref (custom refs don't have remote-tracking refs). +// +// remote is the git remote name, URL, or local path to check against. +// Returns (false, nil) if either ref doesn't exist or they share ancestry. +func IsV2MainDisconnected(ctx context.Context, repo *git.Repository, remote string) (bool, error) { + refName := plumbing.ReferenceName(paths.V2MainRefName) + + localRef, err := repo.Reference(refName, true) + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("failed to check local v2 /main ref: %w", err) + } + + repoPath, err := getRepoPath(repo) + if err != nil { + return false, err + } + + remoteHash, err := lsRemoteRef(ctx, repoPath, remote, paths.V2MainRefName) + if err != nil { + return false, fmt.Errorf("failed to ls-remote v2 /main: %w", err) + } + if remoteHash == plumbing.ZeroHash { + return false, nil // Remote doesn't have the ref + } + + if localRef.Hash() == remoteHash { + return false, nil + } + + // Fetch remote ref to temporary local ref for merge-base check. + // Use the fetched hash (not ls-remote hash) since the remote may have advanced. + if fetchErr := fetchRefToTemp(ctx, repoPath, remote, paths.V2MainRefName, v2DoctorTmpRef); fetchErr != nil { + return false, fmt.Errorf("failed to fetch remote v2 /main: %w", fetchErr) + } + defer cleanupTmpRef(repo) + + fetchedHash, err := resolveRefHash(repo, v2DoctorTmpRef) + if err != nil { + return false, fmt.Errorf("failed to read fetched v2 /main ref: %w", err) + } + + if localRef.Hash() == fetchedHash { + return false, nil + } + + return isDisconnected(ctx, repoPath, localRef.Hash().String(), fetchedHash.String()) +} + +// ReconcileDisconnectedV2Ref detects and repairs disconnected local/remote +// v2 /main refs. Same strategy as v1: cherry-pick local commits onto remote tip. +// The remote is discovered via git ls-remote and fetched to a temp ref. +// +// remote is the git remote name, URL, or local path. +func ReconcileDisconnectedV2Ref( + ctx context.Context, + repo *git.Repository, + remote string, + w io.Writer, +) error { + refName := plumbing.ReferenceName(paths.V2MainRefName) + + localRef, err := repo.Reference(refName, true) + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("failed to check local v2 /main ref: %w", err) + } + + repoPath, err := getRepoPath(repo) + if err != nil { + return err + } + + remoteHash, err := lsRemoteRef(ctx, repoPath, remote, paths.V2MainRefName) + if err != nil { + return fmt.Errorf("failed to ls-remote v2 /main: %w", err) + } + if remoteHash == plumbing.ZeroHash { + return nil + } + + if localRef.Hash() == remoteHash { + return nil + } + + if fetchErr := fetchRefToTemp(ctx, repoPath, remote, paths.V2MainRefName, v2DoctorTmpRef); fetchErr != nil { + return fmt.Errorf("failed to fetch remote v2 /main: %w", fetchErr) + } + defer cleanupTmpRef(repo) + + // Use the fetched hash (not ls-remote hash) since the remote may have advanced. + fetchedHash, err := resolveRefHash(repo, v2DoctorTmpRef) + if err != nil { + return fmt.Errorf("failed to read fetched v2 /main ref: %w", err) + } + + if localRef.Hash() == fetchedHash { + return nil + } + + disconnected, err := isDisconnected(ctx, repoPath, localRef.Hash().String(), fetchedHash.String()) + if err != nil { + return fmt.Errorf("failed to check v2 /main ancestry: %w", err) + } + if !disconnected { + return nil + } + + fmt.Fprintln(w, "[entire] Detected disconnected v2 /main refs (local and remote share no common ancestor)") + + shallow, err := loadShallowHashes(ctx, repoPath) + if err != nil { + return fmt.Errorf("failed to load shallow boundaries: %w", err) + } + + localCommits, err := collectCommitChain(repo, localRef.Hash(), shallow) + if err != nil { + return fmt.Errorf("failed to collect local commits: %w", err) + } + + var dataCommits []*object.Commit + for _, c := range localCommits { + tree, treeErr := c.Tree() + if treeErr != nil { + return fmt.Errorf("failed to read tree for commit %s: %w", c.Hash.String()[:7], treeErr) + } + if len(tree.Entries) > 0 { + dataCommits = append(dataCommits, c) + } + } + + if len(dataCommits) == 0 { + ref := plumbing.NewHashReference(refName, fetchedHash) + if setErr := repo.Storer.SetReference(ref); setErr != nil { + return fmt.Errorf("failed to reset v2 /main to remote: %w", setErr) + } + fmt.Fprintln(w, "[entire] Done — local had no checkpoint data, reset to remote") + return nil + } + + fmt.Fprintf(w, "[entire] Cherry-picking %d local checkpoint(s) onto remote...\n", len(dataCommits)) + + newTip, err := cherryPickOnto(ctx, repo, fetchedHash, dataCommits, shallow) + if err != nil { + return fmt.Errorf("failed to cherry-pick local commits onto remote: %w", err) + } + + ref := plumbing.NewHashReference(refName, newTip) + if setErr := repo.Storer.SetReference(ref); setErr != nil { + return fmt.Errorf("failed to update v2 /main ref: %w", setErr) + } + + fmt.Fprintln(w, "[entire] Done — all local and remote checkpoints preserved") + return nil +} + +// lsRemoteRef runs git ls-remote and returns the hash for a specific ref. +// Returns plumbing.ZeroHash if the ref doesn't exist on the remote. +func lsRemoteRef(ctx context.Context, repoPath, remoteName, refName string) (plumbing.Hash, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + fetchTarget, err := remote.ResolveFetchTarget(ctx, remoteName) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("resolve fetch target for ls-remote: %w", err) + } + + output, err := remote.LsRemoteInDir(ctx, repoPath, fetchTarget, refName) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("git ls-remote %s failed: %w", remote.RedactURL(fetchTarget), err) + } + + line := strings.TrimSpace(string(output)) + if line == "" { + return plumbing.ZeroHash, nil + } + + parts := strings.Fields(line) + if len(parts) < 2 { + return plumbing.ZeroHash, nil + } + + return plumbing.NewHash(parts[0]), nil +} + +// fetchRefToTemp fetches a remote ref to a temporary local ref for comparison. +func fetchRefToTemp(ctx context.Context, repoPath, remoteName, srcRef, dstRef string) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + fetchTarget, err := remote.ResolveFetchTarget(ctx, remoteName) + if err != nil { + return fmt.Errorf("resolve fetch target for doctor v2 fetch: %w", err) + } + + refspec := fmt.Sprintf("+%s:%s", srcRef, dstRef) + output, err := remote.Fetch(ctx, remote.FetchOptions{ + Remote: fetchTarget, + RefSpecs: []string{refspec}, + NoTags: true, + Unshallow: true, + Dir: repoPath, + }) + if err != nil { + redactedURL := remote.RedactURL(fetchTarget) + msg := strings.TrimSpace(strings.ReplaceAll(string(output), fetchTarget, redactedURL)) + if msg != "" { + return fmt.Errorf("git fetch %s failed: %s: %w", redactedURL, msg, err) + } + return fmt.Errorf("git fetch %s failed: %w", redactedURL, err) + } + return nil +} + +// resolveRefHash reads the commit hash that a ref points to. +func resolveRefHash(repo *git.Repository, refName string) (plumbing.Hash, error) { + ref, err := repo.Reference(plumbing.ReferenceName(refName), true) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("ref %s not found: %w", refName, err) + } + return ref.Hash(), nil +} + +// cleanupTmpRef deletes the temporary ref used by doctor checks. +func cleanupTmpRef(repo *git.Repository) { + _ = repo.Storer.RemoveReference(plumbing.ReferenceName(v2DoctorTmpRef)) //nolint:errcheck // best-effort cleanup +} + // isDisconnected checks if two commits have no common ancestor using git merge-base. // Returns (true, nil) if disconnected, (false, nil) if they share ancestry, // or (false, error) if git merge-base failed for another reason. @@ -220,7 +466,13 @@ func isDisconnected(ctx context.Context, repoPath, hashA, hashB string) (bool, e } // collectCommitChain walks from tip to root following first parent, returns oldest-first. -func collectCommitChain(repo *git.Repository, tip plumbing.Hash) ([]*object.Commit, error) { +// Commits listed in shallow are treated as roots — the walk stops at them without +// traversing into their parents. go-git's repo.CommitObject().ParentHashes does not +// consult .git/shallow on its own, so without this check the walk would stroll past +// shallow boundaries into stale objects left in the pack (e.g., when the remote +// branch has been rebuilt since the last full fetch), producing a phantom chain of +// commits that no longer represent the actual checkpoint history. +func collectCommitChain(repo *git.Repository, tip plumbing.Hash, shallow map[plumbing.Hash]bool) ([]*object.Commit, error) { var chain []*object.Commit current := tip @@ -236,6 +488,11 @@ func collectCommitChain(repo *git.Repository, tip plumbing.Hash) ([]*object.Comm reachedRoot = true break } + if shallow[current] { + // Shallow boundary — treat as a root. + reachedRoot = true + break + } current = commit.ParentHashes[0] } @@ -251,14 +508,52 @@ func collectCommitChain(repo *git.Repository, tip plumbing.Hash) ([]*object.Comm return chain, nil } +// loadShallowHashes returns the commit hashes listed in the repository's +// shallow file, or an empty map if the repository is not shallow. +func loadShallowHashes(ctx context.Context, repoPath string) (map[plumbing.Hash]bool, error) { + cmd := exec.CommandContext(ctx, "git", "rev-parse", "--git-common-dir") + cmd.Dir = repoPath + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git rev-parse --git-common-dir: %w", err) + } + gitDir := strings.TrimSpace(string(out)) + if !filepath.IsAbs(gitDir) { + gitDir = filepath.Join(repoPath, gitDir) + } + // Path is constructed from git's own --git-common-dir output, not user input. + data, err := os.ReadFile(filepath.Join(gitDir, "shallow")) //nolint:gosec // see comment above + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return map[plumbing.Hash]bool{}, nil + } + return nil, fmt.Errorf("read shallow file: %w", err) + } + set := map[plumbing.Hash]bool{} + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + set[plumbing.NewHash(line)] = true + } + return set, nil +} + // cherryPickOnto applies each commit's delta onto base, building a linear chain. // For each commit, it computes the full diff from its parent (additions, modifications, // and deletions), then applies that delta onto the current tip's tree. -func cherryPickOnto(ctx context.Context, repo *git.Repository, base plumbing.Hash, commits []*object.Commit) (plumbing.Hash, error) { +// +// Commits listed in shallow are treated as roots: their delta is computed against +// an empty tree rather than against their (past-the-boundary) parent. Without this, +// a shallow-boundary commit would be diffed against a stale parent tree whose +// objects live in the local pack but no longer represent the actual checkpoint +// history — producing nonsense changes when replayed onto the remote tip. +func cherryPickOnto(ctx context.Context, repo *git.Repository, base plumbing.Hash, commits []*object.Commit, shallow map[plumbing.Hash]bool) (plumbing.Hash, error) { currentTip := base for _, commit := range commits { - changes, err := treeChangesForCherryPick(ctx, repo, commit) + changes, err := treeChangesForCherryPick(ctx, repo, commit, shallow) if err != nil { return plumbing.ZeroHash, err } @@ -288,14 +583,15 @@ func cherryPickOnto(ctx context.Context, repo *git.Repository, base plumbing.Has return currentTip, nil } -func treeChangesForCherryPick(ctx context.Context, repo *git.Repository, commit *object.Commit) ([]checkpoint.TreeChange, error) { +func treeChangesForCherryPick(ctx context.Context, repo *git.Repository, commit *object.Commit, shallow map[plumbing.Hash]bool) ([]checkpoint.TreeChange, error) { commitTree, err := commit.Tree() if err != nil { return nil, fmt.Errorf("failed to get tree for commit %s: %w", commit.Hash, err) } var parentTree *object.Tree - if len(commit.ParentHashes) > 0 { + // Shallow-boundary commits are treated as roots — see cherryPickOnto for why. + if len(commit.ParentHashes) > 0 && !shallow[commit.Hash] { parentCommit, pErr := repo.CommitObject(commit.ParentHashes[0]) if pErr != nil { return nil, fmt.Errorf("failed to get parent commit %s: %w", commit.ParentHashes[0], pErr) diff --git a/cmd/entire/cli/strategy/metadata_reconcile_test.go b/cmd/entire/cli/strategy/metadata_reconcile_test.go index b98fa2c554..97420615bc 100644 --- a/cmd/entire/cli/strategy/metadata_reconcile_test.go +++ b/cmd/entire/cli/strategy/metadata_reconcile_test.go @@ -665,12 +665,103 @@ func TestCollectCommitChain_DepthLimit(t *testing.T) { tip = h } - _, err = collectCommitChain(repo, tip) + _, err = collectCommitChain(repo, tip, nil) require.Error(t, err) assert.Contains(t, err.Error(), "exceeded") assert.Contains(t, err.Error(), "without reaching root") } +// TestCollectCommitChain_StopsAtShallowBoundary verifies that collectCommitChain +// treats commits listed in the shallow set as roots, stopping the walk at the +// boundary even when the boundary commit has a parent SHA recorded in the object +// store. Without this behaviour, a shallow checkpoint repo whose remote v1 was +// rebuilt elsewhere would produce a phantom chain of stale commits. +func TestCollectCommitChain_StopsAtShallowBoundary(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + repo, err := git.PlainInit(dir, false) + require.NoError(t, err) + + emptyTree := &object.Tree{Entries: []object.TreeEntry{}} + treeObj := repo.Storer.NewEncodedObject() + require.NoError(t, emptyTree.Encode(treeObj)) + treeHash, err := repo.Storer.SetEncodedObject(treeObj) + require.NoError(t, err) + + // Build a linear chain of 10 commits. Without shallow, the walk should + // return all 10. With the 4th from the tip marked shallow, it should stop + // at that commit, returning 4 entries (tip + 3 below it, including the + // shallow boundary itself, treated as a root). + var tip plumbing.Hash + hashes := make([]plumbing.Hash, 0, 10) + for i := range 10 { + c := &object.Commit{ + TreeHash: treeHash, + Author: object.Signature{Name: "test", Email: "test@test.com", When: time.Now().Add(time.Duration(i) * time.Second)}, + Committer: object.Signature{Name: "test", Email: "test@test.com", When: time.Now().Add(time.Duration(i) * time.Second)}, + Message: "commit\n", + } + if tip != plumbing.ZeroHash { + c.ParentHashes = []plumbing.Hash{tip} + } + obj := repo.Storer.NewEncodedObject() + require.NoError(t, c.Encode(obj)) + h, sErr := repo.Storer.SetEncodedObject(obj) + require.NoError(t, sErr) + hashes = append(hashes, h) + tip = h + } + + // Without shallow set: full chain of 10. + chain, err := collectCommitChain(repo, tip, nil) + require.NoError(t, err) + assert.Len(t, chain, 10, "without shallow, expect full chain") + + // With the 4th-from-tip (index 6 in build order) marked shallow: walk + // stops there. Result is oldest-first: shallow boundary, then up to tip + // = 4 commits. + shallow := map[plumbing.Hash]bool{hashes[6]: true} + chain, err = collectCommitChain(repo, tip, shallow) + require.NoError(t, err) + require.Len(t, chain, 4, "expect tip + 3 commits down to the shallow boundary inclusive") + assert.Equal(t, hashes[6], chain[0].Hash, "oldest entry should be the shallow boundary") + assert.Equal(t, tip, chain[3].Hash, "newest entry should be the tip") +} + +func TestLoadShallowHashes(t *testing.T) { + t.Parallel() + + t.Run("non-shallow repo returns empty set", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + _, err := git.PlainInit(dir, false) + require.NoError(t, err) + + set, err := loadShallowHashes(context.Background(), dir) + require.NoError(t, err) + assert.Empty(t, set) + }) + + t.Run("reads .git/shallow hash list", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + _, err := git.PlainInit(dir, false) + require.NoError(t, err) + + shallowFile := filepath.Join(dir, ".git", "shallow") + require.NoError(t, os.WriteFile(shallowFile, + []byte("be156aa7cc38c2c6117246cb8adad068a3886351\n82b2a8554cccd19f5aa60f520f645b32d8bc2400\n"), + 0o644)) + + set, err := loadShallowHashes(context.Background(), dir) + require.NoError(t, err) + assert.Len(t, set, 2) + assert.True(t, set[plumbing.NewHash("be156aa7cc38c2c6117246cb8adad068a3886351")]) + assert.True(t, set[plumbing.NewHash("82b2a8554cccd19f5aa60f520f645b32d8bc2400")]) + }) +} + // TestReconcileDisconnected_AllEmptyOrphans verifies that when all local commits // are empty-tree orphan commits (the exact bug artifact), reconciliation resets // the local branch to the remote tip without cherry-picking. diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index a5f2e91b7b..8dd8c8b612 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -327,9 +327,12 @@ func fetchAndRebaseSessionsCommon(ctx context.Context, target, branchName string } // Use git CLI for fetch (go-git's fetch can be tricky with auth). - // Use --filter=blob:none for a partial fetch that downloads only commits - // and trees, skipping blobs. The merge only needs the tree structure to - // combine entries; blobs are already local or fetched on demand. + // Do NOT --unshallow here: on a shallow repo with deep history (e.g. a + // shared monorepo), --unshallow downloads the whole repository because + // git treats shallow as a global property of the clone, not per-ref. + // The downstream reconcile/rebase paths walk only commits visible past + // .git/shallow (collectCommitChain / collectCommitsSince), so the + // missing pre-shallow history isn't needed to produce a correct rebase. if output, fetchErr := remote.Fetch(ctx, remote.FetchOptions{ Remote: fetchTarget, RefSpecs: []string{refSpec}, @@ -413,7 +416,12 @@ func fetchAndRebaseSessionsCommon(ctx context.Context, target, branchName string return nil } - newTip, err := cherryPickOnto(ctx, repo, remoteRef.Hash(), localCommits) + shallow, err := loadShallowHashes(ctx, repoPath) + if err != nil { + return fmt.Errorf("failed to load shallow boundaries: %w", err) + } + + newTip, err := cherryPickOnto(ctx, repo, remoteRef.Hash(), localCommits, shallow) if err != nil { return fmt.Errorf("failed to rebase local commits onto remote: %w", err) }