From 31ee2b02217b2f8130037e37111ef09234778efe Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 17 Jun 2026 15:35:36 -0700 Subject: [PATCH 1/5] X-Smart-Branch-Parent: main From 9513be070fdb008702613c7f7ce22b0eb41231f4 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 17 Jun 2026 15:37:08 -0700 Subject: [PATCH 2/5] feat: cache cloned repos in ~/.cache/harness-openshell/repos/ Repos are cached persistently so subsequent runs only fetch deltas instead of re-cloning. On cache hit: git fetch --depth 1, checkout the ref, update submodules, clean untracked files. On cache miss: fresh shallow clone with submodules. Cleanup function is a no-op since the cache is persistent. --- cmd/executor.go | 97 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 16 deletions(-) diff --git a/cmd/executor.go b/cmd/executor.go index 6f099b8..117bbd1 100644 --- a/cmd/executor.go +++ b/cmd/executor.go @@ -166,41 +166,106 @@ func upLocal(opts upLocalOpts) error { return nil } -// cloneRepo clones a git repository to a temp directory and returns an Upload -// that places it at /sandbox/. The clone happens outside the sandbox -// so git credentials never enter it. Returns a cleanup function that removes -// the temp directory. +// cloneRepo clones or updates a cached git repository and returns an Upload +// that places it at /sandbox/. Repos are cached in +// ~/.cache/harness-openshell/repos// so subsequent runs only fetch +// deltas. The clone happens outside the sandbox so git credentials never enter +// it. Returns a cleanup function (no-op since the cache is persistent). func cloneRepo(repo, ref string) (gateway.Upload, func(), error) { repoName := strings.TrimSuffix(path.Base(repo), ".git") - tmpDir, err := os.MkdirTemp("", "harness-repo-") - if err != nil { - return gateway.Upload{}, nil, fmt.Errorf("creating temp dir: %w", err) - } - cleanup := func() { os.RemoveAll(tmpDir) } - cloneDir := filepath.Join(tmpDir, repoName) if ref != "" { status.Infof("Repo: %s (ref: %s)", repo, ref) } else { status.Infof("Repo: %s", repo) } + cacheDir, err := repoCacheDir(repoName) + if err != nil { + return gateway.Upload{}, nil, err + } + + if isGitRepo(cacheDir) { + if err := fetchRepo(cacheDir, ref); err != nil { + return gateway.Upload{}, nil, err + } + status.OKf("Updated %s (cached)", repoName) + } else { + if err := freshClone(repo, ref, cacheDir); err != nil { + return gateway.Upload{}, nil, fmt.Errorf("git clone %s: %w", repo, err) + } + status.OKf("Cloned %s", repoName) + } + + return gateway.Upload{Src: cacheDir, Dst: "/sandbox"}, func() {}, nil +} + +func repoCacheDir(repoName string) (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("determining home dir: %w", err) + } + dir := filepath.Join(home, ".cache", "harness-openshell", "repos", repoName) + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + return "", fmt.Errorf("creating cache dir: %w", err) + } + return dir, nil +} + +func isGitRepo(dir string) bool { + _, err := os.Stat(filepath.Join(dir, ".git")) + return err == nil +} + +func freshClone(repo, ref, dest string) error { args := []string{"clone", "--depth", "1", "--recurse-submodules", "--shallow-submodules"} if ref != "" { args = append(args, "--branch", ref) } - args = append(args, repo, cloneDir) - + args = append(args, repo, dest) cmd := exec.Command("git", args...) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr + return cmd.Run() +} + +func fetchRepo(dir, ref string) error { + fetchArgs := []string{"-C", dir, "fetch", "--depth", "1", "origin"} + if ref != "" { + fetchArgs = append(fetchArgs, ref) + } + cmd := exec.Command("git", fetchArgs...) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("git fetch: %w", err) + } + + target := "FETCH_HEAD" + if ref == "" { + target = "origin/HEAD" + } + cmd = exec.Command("git", "-C", dir, "checkout", target, "--force") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("git checkout %s: %w", target, err) + } + + cmd = exec.Command("git", "-C", dir, "submodule", "update", "--init", "--recursive", "--depth", "1") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - cleanup() - return gateway.Upload{}, nil, fmt.Errorf("git clone %s: %w", repo, err) + return fmt.Errorf("git submodule update: %w", err) } - status.OKf("Cloned %s", repoName) - return gateway.Upload{Src: cloneDir, Dst: "/sandbox"}, cleanup, nil + // Clean untracked files from previous runs + cmd = exec.Command("git", "-C", dir, "clean", "-fdx") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + cmd.Run() + + return nil } var inferenceProviders = map[string]bool{ From 2b6bd8de3283b0db9bf1d482e286430da777307c Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 17 Jun 2026 15:40:17 -0700 Subject: [PATCH 3/5] fix: limit submodule init to one level deep Drop --recursive from submodule init. One level of submodules is sufficient for most repos and avoids pulling deeply nested submodule trees (e.g., third-party vendored dependencies). --- cmd/executor.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/cmd/executor.go b/cmd/executor.go index 117bbd1..d63a5b9 100644 --- a/cmd/executor.go +++ b/cmd/executor.go @@ -218,7 +218,7 @@ func isGitRepo(dir string) bool { } func freshClone(repo, ref, dest string) error { - args := []string{"clone", "--depth", "1", "--recurse-submodules", "--shallow-submodules"} + args := []string{"clone", "--depth", "1"} if ref != "" { args = append(args, "--branch", ref) } @@ -226,7 +226,10 @@ func freshClone(repo, ref, dest string) error { cmd := exec.Command("git", args...) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr - return cmd.Run() + if err := cmd.Run(); err != nil { + return err + } + return initSubmodules(dest) } func fetchRepo(dir, ref string) error { @@ -252,11 +255,8 @@ func fetchRepo(dir, ref string) error { return fmt.Errorf("git checkout %s: %w", target, err) } - cmd = exec.Command("git", "-C", dir, "submodule", "update", "--init", "--recursive", "--depth", "1") - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("git submodule update: %w", err) + if err := initSubmodules(dir); err != nil { + return err } // Clean untracked files from previous runs @@ -268,6 +268,16 @@ func fetchRepo(dir, ref string) error { return nil } +func initSubmodules(dir string) error { + cmd := exec.Command("git", "-C", dir, "submodule", "update", "--init", "--depth", "1") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("git submodule update: %w", err) + } + return nil +} + var inferenceProviders = map[string]bool{ "google-vertex-ai": true, } From 60e9127dcbd5ecc3b14db0915c3606eb7c4b0756 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 17 Jun 2026 15:53:09 -0700 Subject: [PATCH 4/5] fix: CI local job uses Dockerfile path instead of registry pull MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set HARNESS_OS_IMAGE to the Dockerfile directory so openshell builds the sandbox image locally from the Dockerfile instead of pulling from the registry. This avoids the race between images.yml pushing and integration.yml pulling, and avoids the Docker/Podman image store mismatch on CI runners. Also reverts test-local Makefile target to not build — CI handles the image via HARNESS_OS_IMAGE, local dev uses the version-matched registry tag or HARNESS_OS_IMAGE override. --- .github/workflows/integration.yml | 4 ++++ Makefile | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 72d59d3..1d5ba9f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -13,6 +13,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-go@v5 with: go-version-file: go.mod @@ -31,6 +33,7 @@ jobs: - name: Run local integration run: make test-local env: + HARNESS_OS_IMAGE: profiles/images/sandbox-default TEST_LOG_FILE: /tmp/test-logs/test-flow-local.log - name: Run config test suite (with live tests) @@ -81,6 +84,7 @@ jobs: - name: Run kind integration run: make test-kind env: + CONTAINER_CLI: docker TEST_LOG_FILE: /tmp/test-logs/test-flow-kind.log - name: Export logs diff --git a/Makefile b/Makefile index 41155ae..fa05c88 100644 --- a/Makefile +++ b/Makefile @@ -66,10 +66,10 @@ test-suite-live: cli ./test/suite/run.sh --live ## Local gateway integration (unit tests run separately via 'make test') -## Builds sandbox image locally — no registry push needed for Podman. +## Set HARNESS_OS_IMAGE to override the sandbox image. On local dev, +## build first with: make dev-sandbox test-local: cli - $(CONTAINER_CLI) build -t $(IMAGE) profiles/images/sandbox-default/ - HARNESS_OS_IMAGE=$(IMAGE) ./test/test-flow.sh local-container + ./test/test-flow.sh local-container ## Kind: self-contained cluster lifecycle ## Builds sandbox image locally and pre-loads into kind (no registry push needed). From dbf77c8437e600def3f86f90fa1133d16336ca00 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 17 Jun 2026 16:01:59 -0700 Subject: [PATCH 5/5] fix: test-local pushes image on macOS, CI uses Dockerfile path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit macOS: Podman gateway can't see Docker-built images. test-local now depends on dev-push so the image is in the registry. CI (Linux): Docker gateway and Docker build share the same daemon. CI uses HARNESS_OS_IMAGE=profiles/images/sandbox-default so openshell builds from the Dockerfile directly — no registry pull needed. --- .github/workflows/integration.yml | 7 +++---- Makefile | 5 ++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1d5ba9f..947dfa9 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -13,8 +13,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - uses: actions/setup-go@v5 with: go-version-file: go.mod @@ -31,9 +29,10 @@ jobs: openshell gateway list - name: Run local integration - run: make test-local + run: | + CGO_ENABLED=0 go build -ldflags '-s -w -X main.version=ci' -o harness . + HARNESS_OS_IMAGE=profiles/images/sandbox-default ./test/test-flow.sh local-container env: - HARNESS_OS_IMAGE: profiles/images/sandbox-default TEST_LOG_FILE: /tmp/test-logs/test-flow-local.log - name: Run config test suite (with live tests) diff --git a/Makefile b/Makefile index fa05c88..1d1f021 100644 --- a/Makefile +++ b/Makefile @@ -66,9 +66,8 @@ test-suite-live: cli ./test/suite/run.sh --live ## Local gateway integration (unit tests run separately via 'make test') -## Set HARNESS_OS_IMAGE to override the sandbox image. On local dev, -## build first with: make dev-sandbox -test-local: cli +## Builds and pushes the sandbox image so the gateway can pull it. +test-local: cli dev-push ./test/test-flow.sh local-container ## Kind: self-contained cluster lifecycle