From 06e4631c6ab46abff321c7b085ce43c1f2da804b Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 20 May 2026 14:23:23 -0400 Subject: [PATCH 1/7] chore(ci): prototype Swatinem/rust-cache for target/ + cargo caching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the hand-rolled actions/cache step (which only cached ~/.cargo/registry) with Swatinem/rust-cache@v2.9.1, which also caches target/ build artifacts and prunes them intelligently before save: removes incremental artifacts, deps no longer in Cargo.lock, artifacts older than ~1 week, and ~/.cargo/registry/src (recreated from archives on restore). Configuration: - shared-key: "vector-" — single OS-shared cache instead of one per Cargo.lock hash (today's pattern produces ~26 active entries at ~400 MB each, exhausting the 10 GB GHA cache budget). - save-if: refs/heads/master — only master pushes save the cache. PR jobs restore from master's cache but don't write, so PR churn no longer creates per-PR cache entries. Expected behavior on a PR with src/ changes only: Cargo fingerprints match for all unchanged third-party crates (rdkafka-sys, openssl-sys, zstd-sys, …), so only the touched workspace members recompile. Cold build time on cached PRs should drop from ~20m to ~3-5m. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/actions/setup/action.yml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 3719771e4e521..4d90b1470cf76 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -134,17 +134,24 @@ runs: run: echo "::add-matcher::.github/matchers/rust.json" shell: bash - - name: Cache Cargo registry, index, and git DB + # Caches ~/.cargo (registry + git + cargo-installed bin) AND target/ build + # artifacts. Master pushes save the cache; PRs read it but don't write + # back, so a single fixed key per OS keeps the 10 GB GHA cache budget + # under control even as Cargo.lock churns. PRs only recompile what their + # changes invalidate (Cargo fingerprints), so unchanged deps + # (rdkafka-sys, openssl-sys, …) are reused as-is from master's cache. + # + # `key:` is fixed (no lockfile hash) on purpose: Swatinem's default key + # would create a new ~2-3 GB entry per Cargo.lock variant, which blows + # past the 10 GB GHA cap fast. Cargo's fingerprint system handles + # invalidation inside the cache. Bump the `-v?` suffix to force a + # full rebuild if a cache ever gets into a bad state. + - name: Cache Cargo + target/ if: ${{ inputs.cargo-cache == 'true' || env.NEEDS_RUST == 'true' }} - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- + key: "vector-${{ runner.os }}-v1" + save-if: ${{ github.ref == 'refs/heads/master' }} - name: Install mold if: ${{ runner.os == 'Linux' && env.DISABLE_MOLD != 'true' && inputs.mold == 'true' }} From 2e02c05a02b9711f8d7d6175d267990ee759ec07 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 20 May 2026 14:44:17 -0400 Subject: [PATCH 2/7] chore(ci): temp - save cache on all refs to seed prototype So this draft branch can produce a measurable cache hit on its own second push. Revert before merging. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/actions/setup/action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 4d90b1470cf76..55d2ce16a16bb 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -151,7 +151,10 @@ runs: uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: key: "vector-${{ runner.os }}-v1" - save-if: ${{ github.ref == 'refs/heads/master' }} + # TEMP: save on all refs while prototyping so this branch seeds the + # cache. Restore to `${{ github.ref == 'refs/heads/master' }}` before + # merging. + save-if: true - name: Install mold if: ${{ runner.os == 'Linux' && env.DISABLE_MOLD != 'true' && inputs.mold == 'true' }} From d9cb275505035f4f236d957785a6e9dae4f1ed46 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 20 May 2026 14:47:07 -0400 Subject: [PATCH 3/7] chore(ci): use shared-key (key was wrong semantically) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `key:` is additive — Swatinem appends rustc-hash + lockfile-hash to it regardless. `shared-key:` is the documented way to partition only by OS (still gets the rustc/lockfile suffixes appended, but that's the intended default behavior). Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/actions/setup/action.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 55d2ce16a16bb..83b330692feda 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -136,21 +136,20 @@ runs: # Caches ~/.cargo (registry + git + cargo-installed bin) AND target/ build # artifacts. Master pushes save the cache; PRs read it but don't write - # back, so a single fixed key per OS keeps the 10 GB GHA cache budget - # under control even as Cargo.lock churns. PRs only recompile what their - # changes invalidate (Cargo fingerprints), so unchanged deps - # (rdkafka-sys, openssl-sys, …) are reused as-is from master's cache. + # back. PRs only recompile what their changes invalidate (Cargo + # fingerprints), so unchanged deps (rdkafka-sys, openssl-sys, …) are + # reused as-is from master's cache. # - # `key:` is fixed (no lockfile hash) on purpose: Swatinem's default key - # would create a new ~2-3 GB entry per Cargo.lock variant, which blows - # past the 10 GB GHA cap fast. Cargo's fingerprint system handles - # invalidation inside the cache. Bump the `-v?` suffix to force a - # full rebuild if a cache ever gets into a bad state. + # `shared-key` partitions by OS. Swatinem still appends the rustc hash + # and Cargo.lock hash, so a new ~2-3 GB entry is created per Cargo.lock + # change. With master-only saves and LRU eviction of old entries, this + # should keep the 10 GB GHA cache budget under control; if not, consider + # `add-rust-environment-hash-key: false` to drop those suffixes. - name: Cache Cargo + target/ if: ${{ inputs.cargo-cache == 'true' || env.NEEDS_RUST == 'true' }} uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - key: "vector-${{ runner.os }}-v1" + shared-key: "vector-${{ runner.os }}" # TEMP: save on all refs while prototyping so this branch seeds the # cache. Restore to `${{ github.ref == 'refs/heads/master' }}` before # merging. From 62755b5141c12d891f9312694e3a17923c43ceb3 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 20 May 2026 14:50:35 -0400 Subject: [PATCH 4/7] chore: temp source touch to trigger Rust CI The source-change filter in .github/workflows/changes.yml doesn't include .github/actions/setup/**, so this PR's CI run skipped all the Rust-building workflows that exercise the cache. Touching src/main.rs with a comment flips the source filter on. Revert before merging. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/main.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.rs b/src/main.rs index 64d94f4527bfe..3147f68681538 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,7 @@ #![deny(warnings)] +// touch: trigger source-change filter so CI Rust workflows run for the +// rust-cache exploration PR. Revert before merging. extern crate vector; use std::process::ExitCode; From 58f9daad2459e004e1efa70ae6ebe94b4b464b4b Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 20 May 2026 14:54:50 -0400 Subject: [PATCH 5/7] chore(ci): drop redundant runner.os from shared-key Swatinem appends `${runnerOS}-${runnerArch}` automatically. Including runner.os in shared-key produced keys like `vector-Linux-Linux-x64-...`. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/actions/setup/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 83b330692feda..d154ee5081ead 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -149,7 +149,9 @@ runs: if: ${{ inputs.cargo-cache == 'true' || env.NEEDS_RUST == 'true' }} uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - shared-key: "vector-${{ runner.os }}" + # Swatinem appends `${runnerOS}-${runnerArch}` to the key automatically, + # so no need to repeat `runner.os` here. + shared-key: "vector" # TEMP: save on all refs while prototyping so this branch seeds the # cache. Restore to `${{ github.ref == 'refs/heads/master' }}` before # merging. From a6a1e4e0577a7f3bc0c4e7b1f52d72443855514f Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 20 May 2026 15:13:19 -0400 Subject: [PATCH 6/7] chore(ci): drop shared-key, partition cache per job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit shared-key forced multiple Rust CI jobs (check-fmt, check-clippy, Build, …) to compete for the same cache entry on save. The lightest job (check-fmt) consistently won the reservation lock and persisted its near-empty target/ (~486 MB, mostly registry). Heavier jobs that compiled the full workspace got rejected: Failed to save: Unable to reserve cache with key v0-rust-vector-Linux-x64-..., another job may be creating this cache. Letting Swatinem use its default job-id partitioning produces one entry per job, with no contention. Per-entry size will vary by job (clippy and Build will produce multi-GB entries; deny/fmt stay small). Trade off is more entries vs. correct per-job artifact reuse. Also bumps prefix-key to v1-vector so old shared-key entries don't shadow the new job-partitioned ones during transition. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/actions/setup/action.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index d154ee5081ead..cccc688662e01 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -149,9 +149,12 @@ runs: if: ${{ inputs.cargo-cache == 'true' || env.NEEDS_RUST == 'true' }} uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: - # Swatinem appends `${runnerOS}-${runnerArch}` to the key automatically, - # so no need to repeat `runner.os` here. - shared-key: "vector" + # No `shared-key`: let Swatinem partition by job-id (its default). + # Sharing one key across jobs caused races — the lightest job + # (check-fmt) consistently won the save lock, persisting an + # almost-empty target/ and locking out heavier jobs (check-clippy, + # Build) from saving their actual artifacts. + prefix-key: "v1-vector" # TEMP: save on all refs while prototyping so this branch seeds the # cache. Restore to `${{ github.ref == 'refs/heads/master' }}` before # merging. From f7a833e15741ad6921041bccb434ed2608edb504 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 20 May 2026 16:01:21 -0400 Subject: [PATCH 7/7] chore: touch bump to trigger second CI run for cache hit test Confirms whether check-clippy / tests cache restores actually hit (i.e. saved entries from prior run survive eviction). Will revert along with the source touch before merging. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 3147f68681538..04bd2defece2e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ #![deny(warnings)] // touch: trigger source-change filter so CI Rust workflows run for the -// rust-cache exploration PR. Revert before merging. +// rust-cache exploration PR. Revert before merging. (bump 2) extern crate vector; use std::process::ExitCode;