diff --git a/internal/githubapp/config.go b/internal/githubapp/config.go index fb2bd4da..faf94664 100644 --- a/internal/githubapp/config.go +++ b/internal/githubapp/config.go @@ -18,9 +18,12 @@ type TokenCacheConfig struct { } // DefaultTokenCacheConfig returns default token cache configuration. +// RefreshBuffer must exceed the longest subprocess that bakes a token into +// its environment, so a token can't expire mid-call (GitHub App tokens have +// a fixed 1 h TTL). func DefaultTokenCacheConfig() TokenCacheConfig { return TokenCacheConfig{ - RefreshBuffer: 5 * time.Minute, + RefreshBuffer: 30 * time.Minute, JWTExpiration: 10 * time.Minute, } } diff --git a/internal/strategy/git/snapshot.go b/internal/strategy/git/snapshot.go index 214a67fe..28b13896 100644 --- a/internal/strategy/git/snapshot.go +++ b/internal/strategy/git/snapshot.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "sync" + "syscall" "time" "github.com/alecthomas/errors" @@ -23,6 +24,8 @@ import ( "github.com/block/cachew/internal/snapshot" ) +const lfsFetchTimeout = 25 * time.Minute + func snapshotDirForURL(mirrorRoot, upstreamURL string) (string, error) { repoPath, err := gitclone.RepoPathFromURL(upstreamURL) if err != nil { @@ -915,15 +918,28 @@ func (s *Strategy) generateAndUploadLFSSnapshot(ctx context.Context, repo *gitcl } // Fetch only the LFS objects referenced by HEAD (the default branch). + // Timeout must stay below githubapp.RefreshBuffer (30m) so the baked-in + // token can't expire mid-fetch and trigger a retry storm. fetchStart := time.Now() - fetchCmd, err := repo.GitCommand(ctx, "-C", workDir, "lfs", "fetch", "origin", "HEAD") + fetchCtx, cancel := context.WithTimeout(ctx, lfsFetchTimeout) + fetchCmd, err := repo.GitCommand(fetchCtx, "-C", workDir, "lfs", "fetch", "origin", "HEAD") if err != nil { + cancel() s.metrics.recordLFSPhase(ctx, upstream, "fetch", "error", time.Since(fetchStart)) return errors.Wrap(err, "create git lfs fetch command") } - if output, err := fetchCmd.CombinedOutput(); err != nil { + // git-lfs spawns transfer helpers that inherit our pipes; without + // killing the whole group, CombinedOutput stays blocked after the + // timeout fires. + fetchCmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + fetchCmd.Cancel = func() error { + return syscall.Kill(-fetchCmd.Process.Pid, syscall.SIGKILL) + } + output, fetchErr := fetchCmd.CombinedOutput() + cancel() + if fetchErr != nil { s.metrics.recordLFSPhase(ctx, upstream, "fetch", "error", time.Since(fetchStart)) - return errors.Wrapf(err, "git lfs fetch: %s", string(output)) + return errors.Wrapf(fetchErr, "git lfs fetch: %s", string(output)) } s.metrics.recordLFSPhase(ctx, upstream, "fetch", "success", time.Since(fetchStart))