From 5d22df71c71cf9da08adbf34d1b8044f3282cd0a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 06:26:34 +0000 Subject: [PATCH 01/12] Add shared agentic logs cache Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/ab-testing-advisor.lock.yml | 20 +++++ .github/workflows/agentic-logs-cache.yml | 82 +++++++++++++++++++ .../workflows/agentic-token-audit.lock.yml | 20 +++++ .../agentic-token-optimizer.lock.yml | 20 +++++ .../daily-ambient-context-optimizer.lock.yml | 20 +++++ .../workflows/daily-cli-tools-tester.lock.yml | 20 +++++ .github/workflows/daily-evals-report.lock.yml | 20 +++++ .../daily-safe-output-optimizer.lock.yml | 20 +++++ .../daily-security-observability.lock.yml | 26 +++++- .../workflows/daily-security-observability.md | 4 +- .github/workflows/portfolio-analyst.lock.yml | 20 +++++ .../prompt-clustering-analysis.lock.yml | 24 +++++- .../workflows/prompt-clustering-analysis.md | 6 +- .github/workflows/safe-output-health.lock.yml | 20 +++++ pkg/cli/logs_artifact_set.go | 25 ++++++ pkg/cli/logs_artifact_set_test.go | 35 ++++++++ pkg/cli/logs_download.go | 22 ++++- pkg/workflow/compiler_logs_cache.go | 48 +++++++++++ pkg/workflow/compiler_logs_cache_test.go | 62 ++++++++++++++ pkg/workflow/compiler_yaml_runtime_setup.go | 2 + 20 files changed, 504 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/agentic-logs-cache.yml create mode 100644 pkg/workflow/compiler_logs_cache.go create mode 100644 pkg/workflow/compiler_logs_cache_test.go diff --git a/.github/workflows/ab-testing-advisor.lock.yml b/.github/workflows/ab-testing-advisor.lock.yml index 17e8c0e5209..ca1fc242d37 100644 --- a/.github/workflows/ab-testing-advisor.lock.yml +++ b/.github/workflows/ab-testing-advisor.lock.yml @@ -506,6 +506,26 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + path: .github/aw/logs + - name: Enforce shared agentic logs cache TTL + shell: bash + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/agentic-logs-cache.yml b/.github/workflows/agentic-logs-cache.yml new file mode 100644 index 00000000000..513c306841a --- /dev/null +++ b/.github/workflows/agentic-logs-cache.yml @@ -0,0 +1,82 @@ +name: Agentic logs cache + +on: + schedule: + - cron: "0 9 * * *" + workflow_dispatch: + +permissions: + actions: read + contents: read + +jobs: + refresh: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + + - name: Restore latest logs cache + id: restore-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .github/aw/logs + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + + - name: Enforce two-day cache TTL + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi + + - name: Build gh-aw + run: make build + + - name: Refresh 30 days of logs + id: refresh + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p .github/aw/logs + ./gh-aw logs \ + --start-date -30d \ + --count 1000 \ + --artifacts activation,usage \ + --cache-before -30d \ + --output .github/aw/logs \ + --json > /tmp/agentic-logs-summary.json + date -u +%Y-%m-%dT%H:%M:%SZ > .github/aw/logs/.cache-refreshed-at + + - name: Save refreshed logs cache + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .github/aw/logs + key: agentic-logs-${{ github.run_id }} + + - name: Write summary + if: always() + run: | + { + echo "### Agentic logs cache" + if [ -s /tmp/agentic-logs-summary.json ]; then + echo "- Runs cached: $(jq -r '.runs | length' /tmp/agentic-logs-summary.json)" + echo "- Artifacts: activation, usage" + echo "- Window: 30 days" + echo "- Cache TTL: 2 days" + else + echo "- Cache refresh failed before producing a summary." + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/agentic-token-audit.lock.yml b/.github/workflows/agentic-token-audit.lock.yml index 174c56f11fb..b03ec07000c 100644 --- a/.github/workflows/agentic-token-audit.lock.yml +++ b/.github/workflows/agentic-token-audit.lock.yml @@ -536,6 +536,26 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: true run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + path: .github/aw/logs + - name: Enforce shared agentic logs cache TTL + shell: bash + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: diff --git a/.github/workflows/agentic-token-optimizer.lock.yml b/.github/workflows/agentic-token-optimizer.lock.yml index fedc7e49662..e3dc65cd2dc 100644 --- a/.github/workflows/agentic-token-optimizer.lock.yml +++ b/.github/workflows/agentic-token-optimizer.lock.yml @@ -501,6 +501,26 @@ jobs: MEMORY_DIR: /tmp/gh-aw/repo-memory/default CREATE_ORPHAN: true run: bash "${RUNNER_TEMP}/gh-aw/actions/clone_repo_memory_branch.sh" + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + path: .github/aw/logs + - name: Enforce shared agentic logs cache TTL + shell: bash + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Download recent agentic workflow logs diff --git a/.github/workflows/daily-ambient-context-optimizer.lock.yml b/.github/workflows/daily-ambient-context-optimizer.lock.yml index bcec36addd0..189570e98ce 100644 --- a/.github/workflows/daily-ambient-context-optimizer.lock.yml +++ b/.github/workflows/daily-ambient-context-optimizer.lock.yml @@ -544,6 +544,26 @@ jobs: with: name: activation path: /tmp/gh-aw + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + path: .github/aw/logs + - name: Enforce shared agentic logs cache TTL + shell: bash + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: diff --git a/.github/workflows/daily-cli-tools-tester.lock.yml b/.github/workflows/daily-cli-tools-tester.lock.yml index 1ceaa25b398..2c68c23ec35 100644 --- a/.github/workflows/daily-cli-tools-tester.lock.yml +++ b/.github/workflows/daily-cli-tools-tester.lock.yml @@ -523,6 +523,26 @@ jobs: with: name: activation path: /tmp/gh-aw + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + path: .github/aw/logs + - name: Enforce shared agentic logs cache TTL + shell: bash + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-evals-report.lock.yml b/.github/workflows/daily-evals-report.lock.yml index 8ee1ae27ef2..cc1fd456af2 100644 --- a/.github/workflows/daily-evals-report.lock.yml +++ b/.github/workflows/daily-evals-report.lock.yml @@ -529,6 +529,26 @@ jobs: with: name: activation path: /tmp/gh-aw + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + path: .github/aw/logs + - name: Enforce shared agentic logs cache TTL + shell: bash + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-safe-output-optimizer.lock.yml b/.github/workflows/daily-safe-output-optimizer.lock.yml index 347063456af..91d100e769c 100644 --- a/.github/workflows/daily-safe-output-optimizer.lock.yml +++ b/.github/workflows/daily-safe-output-optimizer.lock.yml @@ -600,6 +600,26 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + path: .github/aw/logs + - name: Enforce shared agentic logs cache TTL + shell: bash + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Download logs from last 24 hours diff --git a/.github/workflows/daily-security-observability.lock.yml b/.github/workflows/daily-security-observability.lock.yml index 75df1f2dbb9..0858cdb10cb 100644 --- a/.github/workflows/daily-security-observability.lock.yml +++ b/.github/workflows/daily-security-observability.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"b945a87e6ed3eabaad180c8a3935c5d9264cffd8334b55040726bbca6fee26c9","body_hash":"54a32044c76a0b8e6ab9dc17b949dd123b57dee2a74ca9c4d4117478d3513c46","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"815ca0cd772a20884999d73057a487a0c6e3d43e7d74a11dca158a167c8dabdb","body_hash":"54a32044c76a0b8e6ab9dc17b949dd123b57dee2a74ca9c4d4117478d3513c46","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/setup-python","sha":"5fda3b95a4ea91299a34e894583c3862153e4b97","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"53b7df96c91f9c12dcc8a07bcb9ccacbed38856a","version":"v7.3.0"},{"repo":"docker/setup-buildx-action","sha":"bb05f3f5519dd87d3ba754cc423b652a5edd6d2c","version":"v4.2.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42","digest":"sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42@sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -568,6 +568,26 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + path: .github/aw/logs + - name: Enforce shared agentic logs cache TTL + shell: bash + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi - name: Setup Python environment run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - name: Install Python scientific libraries @@ -607,11 +627,11 @@ jobs: - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Download integrity-filtered logs - run: "mkdir -p /tmp/gh-aw/agent/integrity\nmkdir -p /tmp/gh-aw/cache-memory/security-observability\n\nCACHE_FILE=/tmp/gh-aw/cache-memory/security-observability/filtered-logs.snapshot.json\nRUN_FILE=/tmp/gh-aw/agent/integrity/filtered-logs.json\nFRESH_LOGS=/tmp/gh-aw/agent/integrity/filtered-logs.fresh.json\nEMPTY_DATA='{\"runs\":[],\"summary\":{\"total_runs\":0}}'\nNOW_EPOCH=$(date +%s)\nMAX_CACHE_AGE_SECONDS=$((7 * 24 * 60 * 60))\n\n# Warm start from cached 7-day snapshot when available and fresh.\nif [ -f \"$CACHE_FILE\" ] && jq -e '.runs and .updated_at' \"$CACHE_FILE\" > /dev/null 2>&1; then\n cache_updated_at=$(jq -r '.updated_at' \"$CACHE_FILE\")\n cache_updated_epoch=$(\n date -d \"$cache_updated_at\" +%s 2>/dev/null \\\n || date -j -f \"%Y-%m-%dT%H:%M:%SZ\" \"$cache_updated_at\" +%s 2>/dev/null \\\n || echo 0\n )\n cache_age_seconds=$((NOW_EPOCH - cache_updated_epoch))\n if [ \"$cache_updated_epoch\" -gt 0 ] && [ \"$cache_age_seconds\" -le \"$MAX_CACHE_AGE_SECONDS\" ]; then\n jq '{runs: (.runs // []), summary: (.summary // {\"total_runs\": 0})}' \"$CACHE_FILE\" > \"$RUN_FILE\"\n echo \"✅ Warm cache restored (${cache_age_seconds}s old)\"\n else\n echo \"⚠️ Cache snapshot is stale (${cache_age_seconds}s old); starting fresh\"\n fi\nfi\n\n# Download logs filtered to only runs with DIFC integrity-filtered events.\n# --artifacts mcp: only download the MCP gateway log artifact (sufficient for DIFC checking).\n# --timeout 8: cap execution at 8 minutes to prevent runaway downloads.\ngh aw logs --filtered-integrity --start-date -7d --json -c 200 \\\n --artifacts mcp --timeout 8 \\\n > \"$FRESH_LOGS\" || true\n\n# Validate JSON output and fall back to an empty dataset on failure\nif ! jq -e '.runs' \"$FRESH_LOGS\" > /dev/null 2>&1; then\n echo \"⚠️ No valid logs produced; continuing with empty dataset\"\n echo \"$EMPTY_DATA\" > \"$FRESH_LOGS\"\nfi\n\n# Merge warm-start and fresh runs; fresh entries override warm-cache entries with the same run_id.\nif [ -f \"$RUN_FILE\" ] && jq -e '.runs' \"$RUN_FILE\" > /dev/null 2>&1; then\n jq -s '\n {\n runs: (\n ((.[0].runs // []) + (.[1].runs // []))\n | sort_by(.run_id)\n | group_by(.run_id)\n | map(.[-1])\n ),\n summary: {\n total_runs: (\n ((.[0].runs // []) + (.[1].runs // []) | sort_by(.run_id) | group_by(.run_id) | length)\n )\n }\n }\n ' \"$RUN_FILE\" \"$FRESH_LOGS\" > \"$RUN_FILE.merged\"\n mv \"$RUN_FILE.merged\" \"$RUN_FILE\"\nelse\n mv \"$FRESH_LOGS\" \"$RUN_FILE\"\nfi\n\ncount=$(jq '.runs | length' \"$RUN_FILE\" 2>/dev/null || echo 0)\necho \"✅ Downloaded $count runs with integrity-filtered events\"\n\n# Persist updated 7-day snapshot back to cache-memory every run.\njq -n \\\n --arg updated_at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \\\n --slurpfile payload \"$RUN_FILE\" \\\n '{\n updated_at: $updated_at,\n runs: ($payload[0].runs // []),\n summary: {\n total_runs: (($payload[0].runs // []) | length)\n }\n }' > \"$CACHE_FILE\"\n" + run: "mkdir -p /tmp/gh-aw/agent/integrity\nmkdir -p /tmp/gh-aw/cache-memory/security-observability\n\nCACHE_FILE=/tmp/gh-aw/cache-memory/security-observability/filtered-logs.snapshot.json\nRUN_FILE=/tmp/gh-aw/agent/integrity/filtered-logs.json\nFRESH_LOGS=/tmp/gh-aw/agent/integrity/filtered-logs.fresh.json\nEMPTY_DATA='{\"runs\":[],\"summary\":{\"total_runs\":0}}'\nNOW_EPOCH=$(date +%s)\nMAX_CACHE_AGE_SECONDS=$((7 * 24 * 60 * 60))\n\n# Warm start from cached 7-day snapshot when available and fresh.\nif [ -f \"$CACHE_FILE\" ] && jq -e '.runs and .updated_at' \"$CACHE_FILE\" > /dev/null 2>&1; then\n cache_updated_at=$(jq -r '.updated_at' \"$CACHE_FILE\")\n cache_updated_epoch=$(\n date -d \"$cache_updated_at\" +%s 2>/dev/null \\\n || date -j -f \"%Y-%m-%dT%H:%M:%SZ\" \"$cache_updated_at\" +%s 2>/dev/null \\\n || echo 0\n )\n cache_age_seconds=$((NOW_EPOCH - cache_updated_epoch))\n if [ \"$cache_updated_epoch\" -gt 0 ] && [ \"$cache_age_seconds\" -le \"$MAX_CACHE_AGE_SECONDS\" ]; then\n jq '{runs: (.runs // []), summary: (.summary // {\"total_runs\": 0})}' \"$CACHE_FILE\" > \"$RUN_FILE\"\n echo \"✅ Warm cache restored (${cache_age_seconds}s old)\"\n else\n echo \"⚠️ Cache snapshot is stale (${cache_age_seconds}s old); starting fresh\"\n fi\nfi\n\n# Download logs filtered to only runs with DIFC integrity-filtered events.\n# --artifacts mcp: only download the MCP gateway log artifact (sufficient for DIFC checking).\n# --timeout 8: cap execution at 8 minutes to prevent runaway downloads.\ngh aw logs --filtered-integrity --start-date -7d --json -c 200 \\\n --artifacts mcp --timeout 8 --output .github/aw/logs \\\n > \"$FRESH_LOGS\" || true\n\n# Validate JSON output and fall back to an empty dataset on failure\nif ! jq -e '.runs' \"$FRESH_LOGS\" > /dev/null 2>&1; then\n echo \"⚠️ No valid logs produced; continuing with empty dataset\"\n echo \"$EMPTY_DATA\" > \"$FRESH_LOGS\"\nfi\n\n# Merge warm-start and fresh runs; fresh entries override warm-cache entries with the same run_id.\nif [ -f \"$RUN_FILE\" ] && jq -e '.runs' \"$RUN_FILE\" > /dev/null 2>&1; then\n jq -s '\n {\n runs: (\n ((.[0].runs // []) + (.[1].runs // []))\n | sort_by(.run_id)\n | group_by(.run_id)\n | map(.[-1])\n ),\n summary: {\n total_runs: (\n ((.[0].runs // []) + (.[1].runs // []) | sort_by(.run_id) | group_by(.run_id) | length)\n )\n }\n }\n ' \"$RUN_FILE\" \"$FRESH_LOGS\" > \"$RUN_FILE.merged\"\n mv \"$RUN_FILE.merged\" \"$RUN_FILE\"\nelse\n mv \"$FRESH_LOGS\" \"$RUN_FILE\"\nfi\n\ncount=$(jq '.runs | length' \"$RUN_FILE\" 2>/dev/null || echo 0)\necho \"✅ Downloaded $count runs with integrity-filtered events\"\n\n# Persist updated 7-day snapshot back to cache-memory every run.\njq -n \\\n --arg updated_at \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\" \\\n --slurpfile payload \"$RUN_FILE\" \\\n '{\n updated_at: $updated_at,\n runs: ($payload[0].runs // []),\n summary: {\n total_runs: (($payload[0].runs // []) | length)\n }\n }' > \"$CACHE_FILE\"\n" - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Download firewall-enabled workflow runs - run: "mkdir -p /tmp/gh-aw/agent/firewall\nFIREWALL_RUNS=/tmp/gh-aw/agent/firewall/firewall-enabled-runs.json\nEMPTY_DATA='{\"runs\":[],\"summary\":{\"total_runs\":0}}'\n\n# Download logs filtered to only runs with the firewall feature enabled.\n# --artifacts activation: only download the activation artifact (aw_info.json)\n# needed for firewall detection, avoiding large agent artifact downloads.\n# --timeout 8: cap execution at 8 minutes to prevent runaway downloads.\ngh aw logs --firewall --start-date -7d --json -c 100 \\\n --artifacts activation --timeout 8 \\\n > \"$FIREWALL_RUNS\" || true\n\n# Validate JSON output and fall back to an empty dataset on failure\nif ! jq -e '.runs' \"$FIREWALL_RUNS\" > /dev/null 2>&1; then\n echo \"No valid firewall logs produced; continuing with empty dataset\"\n echo \"$EMPTY_DATA\" > \"$FIREWALL_RUNS\"\nfi\n\ncount=$(jq '.runs | length' \"$FIREWALL_RUNS\" 2>/dev/null || echo 0)\necho \"Downloaded $count firewall-enabled workflow runs\"" + run: "mkdir -p /tmp/gh-aw/agent/firewall\nFIREWALL_RUNS=/tmp/gh-aw/agent/firewall/firewall-enabled-runs.json\nEMPTY_DATA='{\"runs\":[],\"summary\":{\"total_runs\":0}}'\n\n# Download logs filtered to only runs with the firewall feature enabled.\n# --artifacts activation: only download the activation artifact (aw_info.json)\n# needed for firewall detection, avoiding large agent artifact downloads.\n# --timeout 8: cap execution at 8 minutes to prevent runaway downloads.\ngh aw logs --firewall --start-date -7d --json -c 100 \\\n --artifacts activation --timeout 8 --output .github/aw/logs \\\n > \"$FIREWALL_RUNS\" || true\n\n# Validate JSON output and fall back to an empty dataset on failure\nif ! jq -e '.runs' \"$FIREWALL_RUNS\" > /dev/null 2>&1; then\n echo \"No valid firewall logs produced; continuing with empty dataset\"\n echo \"$EMPTY_DATA\" > \"$FIREWALL_RUNS\"\nfi\n\ncount=$(jq '.runs | length' \"$FIREWALL_RUNS\" 2>/dev/null || echo 0)\necho \"Downloaded $count firewall-enabled workflow runs\"" - name: Configure Git credentials env: diff --git a/.github/workflows/daily-security-observability.md b/.github/workflows/daily-security-observability.md index 7dd2def9094..0f002f8e8fe 100644 --- a/.github/workflows/daily-security-observability.md +++ b/.github/workflows/daily-security-observability.md @@ -68,7 +68,7 @@ steps: # --artifacts mcp: only download the MCP gateway log artifact (sufficient for DIFC checking). # --timeout 8: cap execution at 8 minutes to prevent runaway downloads. gh aw logs --filtered-integrity --start-date -7d --json -c 200 \ - --artifacts mcp --timeout 8 \ + --artifacts mcp --timeout 8 --output .github/aw/logs \ > "$FRESH_LOGS" || true # Validate JSON output and fall back to an empty dataset on failure @@ -127,7 +127,7 @@ steps: # needed for firewall detection, avoiding large agent artifact downloads. # --timeout 8: cap execution at 8 minutes to prevent runaway downloads. gh aw logs --firewall --start-date -7d --json -c 100 \ - --artifacts activation --timeout 8 \ + --artifacts activation --timeout 8 --output .github/aw/logs \ > "$FIREWALL_RUNS" || true # Validate JSON output and fall back to an empty dataset on failure diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml index 20d73bd958c..c86acba1297 100644 --- a/.github/workflows/portfolio-analyst.lock.yml +++ b/.github/workflows/portfolio-analyst.lock.yml @@ -517,6 +517,26 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + path: .github/aw/logs + - name: Enforce shared agentic logs cache TTL + shell: bash + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi - name: Setup Python environment run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - name: Install Python scientific libraries diff --git a/.github/workflows/prompt-clustering-analysis.lock.yml b/.github/workflows/prompt-clustering-analysis.lock.yml index 55c3572a74a..484e682382b 100644 --- a/.github/workflows/prompt-clustering-analysis.lock.yml +++ b/.github/workflows/prompt-clustering-analysis.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"70b0c9ce73c234151f94e74ed7289e3296db46e4be2bdb11908aae787da21903","body_hash":"ef6c7a9bc55573a2f261045680c7b96569f47f295d8d736194b3d9191bf77177","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.220"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"fcb6d602a66a3af669a0ae8b2b1504d0655850856f515c3850e3d900e8b4e9f2","body_hash":"ef6c7a9bc55573a2f261045680c7b96569f47f295d8d736194b3d9191bf77177","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.220"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/setup-python","sha":"5fda3b95a4ea91299a34e894583c3862153e4b97","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"53b7df96c91f9c12dcc8a07bcb9ccacbed38856a","version":"v7.3.0"},{"repo":"docker/setup-buildx-action","sha":"bb05f3f5519dd87d3ba754cc423b652a5edd6d2c","version":"v4.2.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42","digest":"sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42@sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -569,6 +569,26 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + path: .github/aw/logs + - name: Enforce shared agentic logs cache TTL + shell: bash + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi - name: Install gh CLI run: | bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" @@ -600,7 +620,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Download workflow logs for PR analysis - run: "# Create logs directory\nmkdir -p /tmp/gh-aw/agent/workflow-logs\n\necho \"Downloading workflow logs to extract turn counts...\"\n\n# Download logs for the last 30 days of copilot workflows\n# This will give us the aw_info.json which contains turn counts\n./gh-aw logs --engine copilot --start-date -30d -o /tmp/gh-aw/agent/workflow-logs\n\n# Verify logs were downloaded\necho \"Downloaded workflow logs:\"\nfind /tmp/gh-aw/agent/workflow-logs -maxdepth 1 -ls" + run: "# Create logs directory\nmkdir -p .github/aw/logs\n\necho \"Downloading workflow logs to extract turn counts...\"\n\n# Download logs for the last 30 days of copilot workflows\n# This will give us the aw_info.json which contains turn counts\n./gh-aw logs --engine copilot --start-date -30d -o .github/aw/logs\n\n# Verify logs were downloaded\necho \"Downloaded workflow logs:\"\nfind .github/aw/logs -maxdepth 1 -ls" # Cache configuration from frontmatter processed below - name: Save prompt clustering data to cache diff --git a/.github/workflows/prompt-clustering-analysis.md b/.github/workflows/prompt-clustering-analysis.md index 29116420c1d..6932804c8db 100644 --- a/.github/workflows/prompt-clustering-analysis.md +++ b/.github/workflows/prompt-clustering-analysis.md @@ -94,17 +94,17 @@ steps: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | # Create logs directory - mkdir -p /tmp/gh-aw/agent/workflow-logs + mkdir -p .github/aw/logs echo "Downloading workflow logs to extract turn counts..." # Download logs for the last 30 days of copilot workflows # This will give us the aw_info.json which contains turn counts - ./gh-aw logs --engine copilot --start-date -30d -o /tmp/gh-aw/agent/workflow-logs + ./gh-aw logs --engine copilot --start-date -30d -o .github/aw/logs # Verify logs were downloaded echo "Downloaded workflow logs:" - find /tmp/gh-aw/agent/workflow-logs -maxdepth 1 -ls + find .github/aw/logs -maxdepth 1 -ls timeout-minutes: 20 features: diff --git a/.github/workflows/safe-output-health.lock.yml b/.github/workflows/safe-output-health.lock.yml index 7b4a4e69a20..a680be7e18e 100644 --- a/.github/workflows/safe-output-health.lock.yml +++ b/.github/workflows/safe-output-health.lock.yml @@ -551,6 +551,26 @@ jobs: GH_AW_CACHE_DIR: /tmp/gh-aw/cache-memory GH_AW_MIN_INTEGRITY: none run: bash "${RUNNER_TEMP}/gh-aw/actions/setup_cache_memory_git.sh" + - name: Restore shared agentic logs cache + id: restore-agentic-logs-cache + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + key: agentic-logs-${{ github.run_id }} + restore-keys: | + agentic-logs- + path: .github/aw/logs + - name: Enforce shared agentic logs cache TTL + shell: bash + run: | + marker=.github/aw/logs/.cache-refreshed-at + if [ -f "$marker" ]; then + refreshed_at=$(cat "$marker") + if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ + || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then + rm -rf .github/aw/logs + fi + fi - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Download logs from last 24 hours diff --git a/pkg/cli/logs_artifact_set.go b/pkg/cli/logs_artifact_set.go index 6aa1f24851f..f4b4fe7baeb 100644 --- a/pkg/cli/logs_artifact_set.go +++ b/pkg/cli/logs_artifact_set.go @@ -14,6 +14,7 @@ package cli import ( "fmt" "os" + "path/filepath" "slices" "sort" "strings" @@ -97,6 +98,8 @@ var artifactSetArtifacts = map[ArtifactSet][]string{ const maxArtifactHintExamples = 2 +const downloadedArtifactsMarkerDir = ".downloaded-artifacts" + // ValidArtifactSetNames returns a sorted list of valid artifact set names, // derived dynamically from the artifactSetArtifacts map to stay in sync automatically. func ValidArtifactSetNames() []string { @@ -254,6 +257,13 @@ func findMissingFilterEntries(filter []string, outputDir string) []string { if e.IsDir() { dirs = append(dirs, e.Name()) } + if markers, markerErr := os.ReadDir(filepath.Join(outputDir, downloadedArtifactsMarkerDir)); markerErr == nil { + for _, marker := range markers { + if !marker.IsDir() { + dirs = append(dirs, marker.Name()) + } + } + } } var missing []string @@ -283,6 +293,21 @@ func findMissingFilterEntries(filter []string, outputDir string) []string { return missing } +func markArtifactDownloaded(outputDir, artifactName string) error { + if artifactName == "" || filepath.Base(artifactName) != artifactName { + return fmt.Errorf("invalid artifact name %q", artifactName) + } + markerDir := filepath.Join(outputDir, downloadedArtifactsMarkerDir) + if err := os.MkdirAll(markerDir, constants.DirPermPublic); err != nil { + return fmt.Errorf("failed to create downloaded artifact marker directory: %w", err) + } + markerPath := filepath.Join(markerDir, artifactName) + if err := os.WriteFile(markerPath, nil, constants.FilePermPublic); err != nil { + return fmt.Errorf("failed to write downloaded artifact marker: %w", err) + } + return nil +} + // applyEvalsArtifact appends the evals artifact set to artifacts when evalsOnly is true // and neither ArtifactSetEvals, ArtifactSetUsage, nor ArtifactSetAll is already present. // Because evals results are now included in the usage artifact, this ensures evals.jsonl diff --git a/pkg/cli/logs_artifact_set_test.go b/pkg/cli/logs_artifact_set_test.go index 8eb4528bbce..a80ebf34e3e 100644 --- a/pkg/cli/logs_artifact_set_test.go +++ b/pkg/cli/logs_artifact_set_test.go @@ -345,6 +345,26 @@ func TestIsUsageOnlyArtifactFilter(t *testing.T) { } } +func TestShouldDownloadWorkflowRunLogs(t *testing.T) { + tests := []struct { + name string + filter []string + expected bool + }{ + {name: "all artifacts", filter: nil, expected: true}, + {name: "usage only", filter: []string{"usage"}, expected: false}, + {name: "activation and usage", filter: []string{"activation", "usage"}, expected: false}, + {name: "agent", filter: []string{"agent"}, expected: true}, + {name: "agent and usage", filter: []string{"agent", "usage"}, expected: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, shouldDownloadWorkflowRunLogs(tt.filter)) + }) + } +} + func TestFindMissingFilterEntries(t *testing.T) { tests := []struct { name string @@ -410,3 +430,18 @@ func TestFindMissingFilterEntries(t *testing.T) { }) } } + +func TestFindMissingFilterEntriesUsesDownloadedMarkers(t *testing.T) { + dir := t.TempDir() + require.NoError(t, markArtifactDownloaded(dir, "activation")) + require.NoError(t, markArtifactDownloaded(dir, "abc123-usage")) + + assert.Nil(t, findMissingFilterEntries([]string{"activation", "usage"}, dir)) + assert.Equal(t, []string{"agent"}, findMissingFilterEntries([]string{"activation", "agent"}, dir)) +} + +func TestMarkArtifactDownloadedRejectsInvalidNames(t *testing.T) { + err := markArtifactDownloaded(t.TempDir(), "../activation") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid artifact name") +} diff --git a/pkg/cli/logs_download.go b/pkg/cli/logs_download.go index 48c210a276a..267e08a05c2 100644 --- a/pkg/cli/logs_download.go +++ b/pkg/cli/logs_download.go @@ -55,6 +55,18 @@ func isUsageOnlyArtifactFilter(artifactFilter []string) bool { return len(artifactFilter) == 1 && artifactFilter[0] == constants.UsageArtifactName } +func shouldDownloadWorkflowRunLogs(artifactFilter []string) bool { + if len(artifactFilter) == 0 { + return true + } + for _, artifact := range artifactFilter { + if artifact != constants.ActivationArtifactName && artifact != constants.UsageArtifactName { + return true + } + } + return false +} + // flattenSingleFileArtifacts checks artifact directories and flattens any that contain a single file // This handles the case where gh CLI creates a directory for each artifact, even if it's just one file func flattenSingleFileArtifacts(outputDir string, verbose bool) error { @@ -612,6 +624,9 @@ func downloadArtifactsByName(ctx context.Context, opts downloadArtifactsOptions, // Non-fatal: continue downloading other artifacts } else { logsDownloadLog.Printf("Downloaded artifact %q", name) + if err := markArtifactDownloaded(opts.outputDir, name); err != nil { + return err + } } } @@ -669,6 +684,9 @@ func retryCriticalArtifacts(ctx context.Context, opts downloadArtifactsOptions) } } else { logsDownloadLog.Printf("Successfully downloaded artifact %q individually", name) + if err := markArtifactDownloaded(opts.outputDir, name); err != nil { + logsDownloadLog.Printf("Failed to mark artifact %q as downloaded: %v", name, err) + } if opts.verbose { fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Downloaded missing artifact: "+name)) } @@ -781,7 +799,7 @@ func downloadRunArtifacts(ctx context.Context, opts downloadArtifactsOptions) er if len(downloadableNames) == 0 { // Nothing to download (all artifacts are either .dockerbuild or excluded by filter). // For usage-only mode, skip workflow logs entirely to keep downloads lightweight. - if !isUsageOnlyArtifactFilter(opts.artifactFilter) { + if shouldDownloadWorkflowRunLogs(opts.artifactFilter) { // Attempt workflow run logs for diagnostics before returning. if logErr := downloadWorkflowRunLogs(ctx, opts.runID, opts.outputDir, opts.verbose, opts.owner, opts.repo, opts.hostname); logErr != nil { if opts.verbose { @@ -956,7 +974,7 @@ func downloadRunArtifacts(ctx context.Context, opts downloadArtifactsOptions) er } // Download and unzip workflow run logs unless caller requested usage-only mode. - if !isUsageOnlyArtifactFilter(opts.artifactFilter) { + if shouldDownloadWorkflowRunLogs(opts.artifactFilter) { if err := downloadWorkflowRunLogs(ctx, opts.runID, opts.outputDir, opts.verbose, opts.owner, opts.repo, opts.hostname); err != nil { // Log the error but don't fail the entire download process // Logs may not be available for all runs (e.g., expired or deleted) diff --git a/pkg/workflow/compiler_logs_cache.go b/pkg/workflow/compiler_logs_cache.go new file mode 100644 index 00000000000..5c9acb9a443 --- /dev/null +++ b/pkg/workflow/compiler_logs_cache.go @@ -0,0 +1,48 @@ +package workflow + +import ( + "fmt" + "strings" +) + +const sharedLogsCachePath = ".github/aw/logs" + +func usesSharedLogsCache(data *WorkflowData) bool { + if !strings.Contains(data.On, "schedule") { + return false + } + content := data.CustomSteps + "\n" + data.MarkdownContent + for _, command := range []string{"gh aw logs", "./gh-aw logs", "gh aw audit", "./gh-aw audit"} { + if strings.Contains(content, command) { + return true + } + } + return false +} + +func generateSharedLogsCacheRestoreSteps(yaml *strings.Builder, data *WorkflowData) { + if !usesSharedLogsCache(data) { + return + } + + yaml.WriteString(" - name: Restore shared agentic logs cache\n") + yaml.WriteString(" id: restore-agentic-logs-cache\n") + yaml.WriteString(" continue-on-error: true\n") + fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/cache/restore", data)) + yaml.WriteString(" with:\n") + yaml.WriteString(" key: agentic-logs-${{ github.run_id }}\n") + yaml.WriteString(" restore-keys: |\n") + yaml.WriteString(" agentic-logs-\n") + fmt.Fprintf(yaml, " path: %s\n", sharedLogsCachePath) + yaml.WriteString(" - name: Enforce shared agentic logs cache TTL\n") + yaml.WriteString(" shell: bash\n") + yaml.WriteString(" run: |\n") + fmt.Fprintf(yaml, " marker=%s/.cache-refreshed-at\n", sharedLogsCachePath) + yaml.WriteString(" if [ -f \"$marker\" ]; then\n") + yaml.WriteString(" refreshed_at=$(cat \"$marker\")\n") + yaml.WriteString(" if ! refreshed_epoch=$(date -d \"$refreshed_at\" +%s 2>/dev/null) \\\n") + yaml.WriteString(" || [ \"$refreshed_epoch\" -lt \"$(date -u -d '2 days ago' +%s)\" ]; then\n") + fmt.Fprintf(yaml, " rm -rf %s\n", sharedLogsCachePath) + yaml.WriteString(" fi\n") + yaml.WriteString(" fi\n") +} diff --git a/pkg/workflow/compiler_logs_cache_test.go b/pkg/workflow/compiler_logs_cache_test.go new file mode 100644 index 00000000000..c9ce1cf2449 --- /dev/null +++ b/pkg/workflow/compiler_logs_cache_test.go @@ -0,0 +1,62 @@ +package workflow + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUsesSharedLogsCache(t *testing.T) { + tests := []struct { + name string + data WorkflowData + want bool + }{ + { + name: "scheduled custom logs command", + data: WorkflowData{On: "schedule: daily", CustomSteps: "run: gh aw logs --json"}, + want: true, + }, + { + name: "scheduled prompt audit command", + data: WorkflowData{On: "schedule: daily", MarkdownContent: "Run `gh aw audit 123`."}, + want: true, + }, + { + name: "non-scheduled logs command", + data: WorkflowData{On: "workflow_dispatch:", CustomSteps: "run: gh aw logs"}, + want: false, + }, + { + name: "scheduled unrelated workflow", + data: WorkflowData{On: "schedule: daily", MarkdownContent: "Review issues."}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, usesSharedLogsCache(&tt.data)) + }) + } +} + +func TestGenerateSharedLogsCacheRestoreSteps(t *testing.T) { + cache := NewActionCache(t.TempDir()) + data := &WorkflowData{ + On: "schedule: daily", + CustomSteps: "run: ./gh-aw logs", + ActionCache: cache, + ActionResolver: NewActionResolver(cache), + } + var yaml strings.Builder + + generateSharedLogsCacheRestoreSteps(&yaml, data) + + output := yaml.String() + assert.Contains(t, output, "actions/cache/restore@") + assert.Contains(t, output, "path: "+sharedLogsCachePath) + assert.Contains(t, output, "2 days ago") + assert.NotContains(t, output, "actions/cache/save@") +} diff --git a/pkg/workflow/compiler_yaml_runtime_setup.go b/pkg/workflow/compiler_yaml_runtime_setup.go index 14483ee5698..00d77484022 100644 --- a/pkg/workflow/compiler_yaml_runtime_setup.go +++ b/pkg/workflow/compiler_yaml_runtime_setup.go @@ -65,6 +65,8 @@ func (c *Compiler) generateRuntimeAndWorkspaceSetupSteps(yaml *strings.Builder, compilerYamlLog.Printf("Generating repo-memory steps for workflow") generateRepoMemorySteps(yaml, data) + generateSharedLogsCacheRestoreSteps(yaml, data) + c.emitCustomSteps(yaml, data, customStepsContainCheckout, runtimeSetupSteps) // Add cache steps if cache configuration is present. Keep workspace caches after user From 8cc5a733b3b8492c078734da1adf8dcc6ed58fba Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 06:41:50 +0000 Subject: [PATCH 02/12] Honor artifact-specific cache lookups Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/ab-testing-advisor.lock.yml | 4 +- .github/workflows/agentic-logs-cache.yml | 4 +- .../workflows/agentic-token-audit.lock.yml | 4 +- .../agentic-token-optimizer.lock.yml | 4 +- .../daily-ambient-context-optimizer.lock.yml | 4 +- .../workflows/daily-cli-tools-tester.lock.yml | 4 +- .github/workflows/daily-evals-report.lock.yml | 4 +- .../daily-safe-output-optimizer.lock.yml | 4 +- .../daily-security-observability.lock.yml | 4 +- .github/workflows/portfolio-analyst.lock.yml | 4 +- .../prompt-clustering-analysis.lock.yml | 4 +- .github/workflows/safe-output-health.lock.yml | 4 +- pkg/cli/logs_run_processor.go | 6 ++ pkg/cli/logs_run_processor_test.go | 41 ++++++++++++ pkg/workflow/compiler_logs_cache.go | 66 +++++++++++-------- pkg/workflow/compiler_logs_cache_test.go | 22 +++++++ pkg/workflow/compiler_yaml_runtime_setup.go | 7 +- 17 files changed, 151 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ab-testing-advisor.lock.yml b/.github/workflows/ab-testing-advisor.lock.yml index ca1fc242d37..16c2462419a 100644 --- a/.github/workflows/ab-testing-advisor.lock.yml +++ b/.github/workflows/ab-testing-advisor.lock.yml @@ -519,7 +519,9 @@ jobs: shell: bash run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/.github/workflows/agentic-logs-cache.yml b/.github/workflows/agentic-logs-cache.yml index 513c306841a..70734a01686 100644 --- a/.github/workflows/agentic-logs-cache.yml +++ b/.github/workflows/agentic-logs-cache.yml @@ -34,7 +34,9 @@ jobs: - name: Enforce two-day cache TTL run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/.github/workflows/agentic-token-audit.lock.yml b/.github/workflows/agentic-token-audit.lock.yml index b03ec07000c..854803c3991 100644 --- a/.github/workflows/agentic-token-audit.lock.yml +++ b/.github/workflows/agentic-token-audit.lock.yml @@ -549,7 +549,9 @@ jobs: shell: bash run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/.github/workflows/agentic-token-optimizer.lock.yml b/.github/workflows/agentic-token-optimizer.lock.yml index e3dc65cd2dc..5dd8a9cbbf0 100644 --- a/.github/workflows/agentic-token-optimizer.lock.yml +++ b/.github/workflows/agentic-token-optimizer.lock.yml @@ -514,7 +514,9 @@ jobs: shell: bash run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/.github/workflows/daily-ambient-context-optimizer.lock.yml b/.github/workflows/daily-ambient-context-optimizer.lock.yml index 189570e98ce..598b8631857 100644 --- a/.github/workflows/daily-ambient-context-optimizer.lock.yml +++ b/.github/workflows/daily-ambient-context-optimizer.lock.yml @@ -557,7 +557,9 @@ jobs: shell: bash run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/.github/workflows/daily-cli-tools-tester.lock.yml b/.github/workflows/daily-cli-tools-tester.lock.yml index 2c68c23ec35..ee09c4ff45f 100644 --- a/.github/workflows/daily-cli-tools-tester.lock.yml +++ b/.github/workflows/daily-cli-tools-tester.lock.yml @@ -536,7 +536,9 @@ jobs: shell: bash run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/.github/workflows/daily-evals-report.lock.yml b/.github/workflows/daily-evals-report.lock.yml index cc1fd456af2..deec5b90bf3 100644 --- a/.github/workflows/daily-evals-report.lock.yml +++ b/.github/workflows/daily-evals-report.lock.yml @@ -542,7 +542,9 @@ jobs: shell: bash run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/.github/workflows/daily-safe-output-optimizer.lock.yml b/.github/workflows/daily-safe-output-optimizer.lock.yml index 91d100e769c..87fbb667fc8 100644 --- a/.github/workflows/daily-safe-output-optimizer.lock.yml +++ b/.github/workflows/daily-safe-output-optimizer.lock.yml @@ -613,7 +613,9 @@ jobs: shell: bash run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/.github/workflows/daily-security-observability.lock.yml b/.github/workflows/daily-security-observability.lock.yml index 0858cdb10cb..c7354d5e484 100644 --- a/.github/workflows/daily-security-observability.lock.yml +++ b/.github/workflows/daily-security-observability.lock.yml @@ -581,7 +581,9 @@ jobs: shell: bash run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml index c86acba1297..a992fc0e7db 100644 --- a/.github/workflows/portfolio-analyst.lock.yml +++ b/.github/workflows/portfolio-analyst.lock.yml @@ -530,7 +530,9 @@ jobs: shell: bash run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/.github/workflows/prompt-clustering-analysis.lock.yml b/.github/workflows/prompt-clustering-analysis.lock.yml index 484e682382b..1acd49f092b 100644 --- a/.github/workflows/prompt-clustering-analysis.lock.yml +++ b/.github/workflows/prompt-clustering-analysis.lock.yml @@ -582,7 +582,9 @@ jobs: shell: bash run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/.github/workflows/safe-output-health.lock.yml b/.github/workflows/safe-output-health.lock.yml index a680be7e18e..df039cd2cb1 100644 --- a/.github/workflows/safe-output-health.lock.yml +++ b/.github/workflows/safe-output-health.lock.yml @@ -564,7 +564,9 @@ jobs: shell: bash run: | marker=.github/aw/logs/.cache-refreshed-at - if [ -f "$marker" ]; then + if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then + rm -rf .github/aw/logs + elif [ -f "$marker" ]; then refreshed_at=$(cat "$marker") if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then diff --git a/pkg/cli/logs_run_processor.go b/pkg/cli/logs_run_processor.go index 96930681589..7575aa8852f 100644 --- a/pkg/cli/logs_run_processor.go +++ b/pkg/cli/logs_run_processor.go @@ -274,6 +274,12 @@ func tryLoadCachedRunResult( if !ok { return nil, false } + if len(params.artifactFilter) > 0 { + if missing := findMissingFilterEntries(params.artifactFilter, runOutputDir); len(missing) > 0 { + logsOrchestratorLog.Printf("Cache bypass for run %d: requested artifacts missing locally: %v", run.DatabaseID, missing) + return nil, false + } + } // When --evals is requested but evals are not present in the cached run directory // (e.g., the run was cached before evals were included in the usage artifact), diff --git a/pkg/cli/logs_run_processor_test.go b/pkg/cli/logs_run_processor_test.go index 9dd5ad90628..3f09030eb4c 100644 --- a/pkg/cli/logs_run_processor_test.go +++ b/pkg/cli/logs_run_processor_test.go @@ -168,6 +168,7 @@ func TestTryLoadCachedRunResultUsesCacheWhenEvalsNotRequested(t *testing.T) { DatabaseID: 124, }, } + require.NoError(t, saveRunSummary(runOutputDir, summary, false)) result, ok := tryLoadCachedRunResult(context.Background(), WorkflowRun{DatabaseID: 124}, runOutputDir, concurrentRunDownloadParams{ @@ -180,6 +181,46 @@ func TestTryLoadCachedRunResultUsesCacheWhenEvalsNotRequested(t *testing.T) { assert.True(t, result.Cached) } +func TestTryLoadCachedRunResultBypassesCacheWhenRequestedArtifactIsMissing(t *testing.T) { + runOutputDir := t.TempDir() + summary := &RunSummary{ + CLIVersion: GetVersion(), + RunID: 125, + ProcessedAt: time.Now(), + Run: WorkflowRun{DatabaseID: 125}, + } + require.NoError(t, saveRunSummary(runOutputDir, summary, false)) + require.NoError(t, markArtifactDownloaded(runOutputDir, "activation")) + + result, ok := tryLoadCachedRunResult(context.Background(), WorkflowRun{DatabaseID: 125}, runOutputDir, concurrentRunDownloadParams{ + artifactFilter: []string{"agent"}, + }) + + assert.False(t, ok) + assert.Nil(t, result) +} + +func TestTryLoadCachedRunResultUsesCacheWhenRequestedArtifactsArePresent(t *testing.T) { + runOutputDir := t.TempDir() + summary := &RunSummary{ + CLIVersion: GetVersion(), + RunID: 126, + ProcessedAt: time.Now(), + Run: WorkflowRun{DatabaseID: 126}, + } + require.NoError(t, saveRunSummary(runOutputDir, summary, false)) + require.NoError(t, markArtifactDownloaded(runOutputDir, "activation")) + require.NoError(t, markArtifactDownloaded(runOutputDir, "usage")) + + result, ok := tryLoadCachedRunResult(context.Background(), WorkflowRun{DatabaseID: 126}, runOutputDir, concurrentRunDownloadParams{ + artifactFilter: []string{"activation", "usage"}, + }) + + require.True(t, ok) + require.NotNil(t, result) + assert.True(t, result.Cached) +} + // TestTryLoadCachedRunResultPersistsSafeItemsCountAfterBackfill verifies that when // tryLoadCachedRunResult heals a stale SafeItemsCount (0 → N) via backfillCacheHitIfNeeded, // the healed value is written back to run_summary.json on disk so downstream readers diff --git a/pkg/workflow/compiler_logs_cache.go b/pkg/workflow/compiler_logs_cache.go index 5c9acb9a443..fbee5211c6e 100644 --- a/pkg/workflow/compiler_logs_cache.go +++ b/pkg/workflow/compiler_logs_cache.go @@ -1,9 +1,6 @@ package workflow -import ( - "fmt" - "strings" -) +import "strings" const sharedLogsCachePath = ".github/aw/logs" @@ -20,29 +17,46 @@ func usesSharedLogsCache(data *WorkflowData) bool { return false } -func generateSharedLogsCacheRestoreSteps(yaml *strings.Builder, data *WorkflowData) { +func sharedLogsCacheRestoreSteps(data *WorkflowData) []GitHubActionStep { if !usesSharedLogsCache(data) { - return + return nil + } + + return []GitHubActionStep{ + { + " - name: Restore shared agentic logs cache", + " id: restore-agentic-logs-cache", + " continue-on-error: true", + " uses: " + getCachedActionPin("actions/cache/restore", data), + " with:", + " key: agentic-logs-${{ github.run_id }}", + " restore-keys: |", + " agentic-logs-", + " path: " + sharedLogsCachePath, + }, + { + " - name: Enforce shared agentic logs cache TTL", + " shell: bash", + " run: |", + " marker=" + sharedLogsCachePath + "/.cache-refreshed-at", + " if [ -d " + sharedLogsCachePath + " ] && [ ! -f \"$marker\" ]; then", + " rm -rf " + sharedLogsCachePath, + " elif [ -f \"$marker\" ]; then", + " refreshed_at=$(cat \"$marker\")", + " if ! refreshed_epoch=$(date -d \"$refreshed_at\" +%s 2>/dev/null) \\", + " || [ \"$refreshed_epoch\" -lt \"$(date -u -d '2 days ago' +%s)\" ]; then", + " rm -rf " + sharedLogsCachePath, + " fi", + " fi", + }, } +} - yaml.WriteString(" - name: Restore shared agentic logs cache\n") - yaml.WriteString(" id: restore-agentic-logs-cache\n") - yaml.WriteString(" continue-on-error: true\n") - fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/cache/restore", data)) - yaml.WriteString(" with:\n") - yaml.WriteString(" key: agentic-logs-${{ github.run_id }}\n") - yaml.WriteString(" restore-keys: |\n") - yaml.WriteString(" agentic-logs-\n") - fmt.Fprintf(yaml, " path: %s\n", sharedLogsCachePath) - yaml.WriteString(" - name: Enforce shared agentic logs cache TTL\n") - yaml.WriteString(" shell: bash\n") - yaml.WriteString(" run: |\n") - fmt.Fprintf(yaml, " marker=%s/.cache-refreshed-at\n", sharedLogsCachePath) - yaml.WriteString(" if [ -f \"$marker\" ]; then\n") - yaml.WriteString(" refreshed_at=$(cat \"$marker\")\n") - yaml.WriteString(" if ! refreshed_epoch=$(date -d \"$refreshed_at\" +%s 2>/dev/null) \\\n") - yaml.WriteString(" || [ \"$refreshed_epoch\" -lt \"$(date -u -d '2 days ago' +%s)\" ]; then\n") - fmt.Fprintf(yaml, " rm -rf %s\n", sharedLogsCachePath) - yaml.WriteString(" fi\n") - yaml.WriteString(" fi\n") +func generateSharedLogsCacheRestoreSteps(yaml *strings.Builder, data *WorkflowData) { + for _, step := range sharedLogsCacheRestoreSteps(data) { + for _, line := range step { + yaml.WriteString(line) + yaml.WriteByte('\n') + } + } } diff --git a/pkg/workflow/compiler_logs_cache_test.go b/pkg/workflow/compiler_logs_cache_test.go index c9ce1cf2449..2597b72acdb 100644 --- a/pkg/workflow/compiler_logs_cache_test.go +++ b/pkg/workflow/compiler_logs_cache_test.go @@ -58,5 +58,27 @@ func TestGenerateSharedLogsCacheRestoreSteps(t *testing.T) { assert.Contains(t, output, "actions/cache/restore@") assert.Contains(t, output, "path: "+sharedLogsCachePath) assert.Contains(t, output, "2 days ago") + assert.Contains(t, output, "[ ! -f \"$marker\" ]") assert.NotContains(t, output, "actions/cache/save@") } + +func TestSharedLogsCacheRestoreFollowsCustomCheckout(t *testing.T) { + cache := NewActionCache(t.TempDir()) + data := &WorkflowData{ + On: "schedule: daily", + CustomSteps: "steps:\n - uses: actions/checkout@v4\n - name: Download logs\n run: gh aw logs", + ActionCache: cache, + ActionResolver: NewActionResolver(cache), + } + var yaml strings.Builder + compiler := &Compiler{} + + compiler.addCustomStepsWithRuntimeInsertion(&yaml, data.CustomSteps, sharedLogsCacheRestoreSteps(data), nil, false) + + output := yaml.String() + checkoutIndex := strings.Index(output, "uses: actions/checkout@v4") + restoreIndex := strings.Index(output, "Restore shared agentic logs cache") + logsIndex := strings.Index(output, "run: gh aw logs") + assert.Greater(t, restoreIndex, checkoutIndex) + assert.Greater(t, logsIndex, restoreIndex) +} diff --git a/pkg/workflow/compiler_yaml_runtime_setup.go b/pkg/workflow/compiler_yaml_runtime_setup.go index 00d77484022..e2b0a7abb60 100644 --- a/pkg/workflow/compiler_yaml_runtime_setup.go +++ b/pkg/workflow/compiler_yaml_runtime_setup.go @@ -21,6 +21,9 @@ import ( // themselves contain a checkout action (used by the caller to compute needsGitConfig). func (c *Compiler) generateRuntimeAndWorkspaceSetupSteps(yaml *strings.Builder, data *WorkflowData, needsCheckout bool) bool { runtimeSetupSteps, customStepsContainCheckout := c.prepareRuntimeSetupAndCheckoutInfo(data) + if customStepsContainCheckout { + runtimeSetupSteps = append(runtimeSetupSteps, sharedLogsCacheRestoreSteps(data)...) + } compilerYamlLog.Printf("Custom steps contain checkout: %t (len(customSteps)=%d)", customStepsContainCheckout, len(data.CustomSteps)) c.emitRuntimeSetupPrelude(yaml, data, needsCheckout, customStepsContainCheckout, runtimeSetupSteps) @@ -65,7 +68,9 @@ func (c *Compiler) generateRuntimeAndWorkspaceSetupSteps(yaml *strings.Builder, compilerYamlLog.Printf("Generating repo-memory steps for workflow") generateRepoMemorySteps(yaml, data) - generateSharedLogsCacheRestoreSteps(yaml, data) + if !customStepsContainCheckout { + generateSharedLogsCacheRestoreSteps(yaml, data) + } c.emitCustomSteps(yaml, data, customStepsContainCheckout, runtimeSetupSteps) From ce1547762706fa1f93d6d274728188db2ff55c0c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 06:53:06 +0000 Subject: [PATCH 03/12] Require complete cache marker for audits Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs_download.go | 27 +++++++++++++++------------ pkg/cli/logs_run_processor.go | 7 ++++++- pkg/cli/logs_run_processor_test.go | 26 ++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/pkg/cli/logs_download.go b/pkg/cli/logs_download.go index 267e08a05c2..5078e9987c8 100644 --- a/pkg/cli/logs_download.go +++ b/pkg/cli/logs_download.go @@ -725,22 +725,20 @@ func downloadRunArtifacts(ctx context.Context, opts downloadArtifactsOptions) er // Fall through to the download code below (MkdirAll is a no-op for existing dir). } else { // No filter — caller wants all artifacts. Keep the existing behaviour: - // if the directory is non-empty we assume the run was previously fully - // downloaded and skip the download. - if summary, ok := loadRunSummary(opts.outputDir, opts.verbose); ok { - // Valid cached summary exists, skip download - logsDownloadLog.Printf("Using cached artifacts for run %d", opts.runID) - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Using cached artifacts for run %d at %s (from %s)", opts.runID, opts.outputDir, summary.ProcessedAt.Format("2006-01-02 15:04:05")))) + // only a complete bulk download marker can satisfy an all-artifacts request. + if len(findMissingFilterEntries([]string{string(ArtifactSetAll)}, opts.outputDir)) == 0 { + if summary, ok := loadRunSummary(opts.outputDir, opts.verbose); ok { + // Valid cached summary exists, skip download + logsDownloadLog.Printf("Using cached artifacts for run %d", opts.runID) + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Using cached artifacts for run %d at %s (from %s)", opts.runID, opts.outputDir, summary.ProcessedAt.Format("2006-01-02 15:04:05")))) + } + return nil } - return nil } - // Summary doesn't exist or version mismatch - artifacts exist but need reprocessing - // Don't re-download, just reprocess what's there if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Run folder exists with artifacts, will reprocess run %d without re-downloading", opts.runID))) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Run folder for %d is missing the complete artifact marker; downloading all artifacts", opts.runID))) } - return nil } } @@ -936,6 +934,11 @@ func downloadRunArtifacts(ctx context.Context, opts downloadArtifactsOptions) er // workflow logs rather than artifact content. return ErrNoArtifacts } + if err == nil { + if markerErr := markArtifactDownloaded(opts.outputDir, string(ArtifactSetAll)); markerErr != nil { + return markerErr + } + } } // Stop spinner with success message diff --git a/pkg/cli/logs_run_processor.go b/pkg/cli/logs_run_processor.go index 7575aa8852f..5a13201137a 100644 --- a/pkg/cli/logs_run_processor.go +++ b/pkg/cli/logs_run_processor.go @@ -274,7 +274,12 @@ func tryLoadCachedRunResult( if !ok { return nil, false } - if len(params.artifactFilter) > 0 { + if len(params.artifactFilter) == 0 { + if missing := findMissingFilterEntries([]string{string(ArtifactSetAll)}, runOutputDir); len(missing) > 0 { + logsOrchestratorLog.Printf("Cache bypass for run %d: complete artifact marker missing", run.DatabaseID) + return nil, false + } + } else { if missing := findMissingFilterEntries(params.artifactFilter, runOutputDir); len(missing) > 0 { logsOrchestratorLog.Printf("Cache bypass for run %d: requested artifacts missing locally: %v", run.DatabaseID, missing) return nil, false diff --git a/pkg/cli/logs_run_processor_test.go b/pkg/cli/logs_run_processor_test.go index 3f09030eb4c..4eaed7cc710 100644 --- a/pkg/cli/logs_run_processor_test.go +++ b/pkg/cli/logs_run_processor_test.go @@ -148,6 +148,7 @@ func TestTryLoadCachedRunResultBypassesForExplicitEvalsArtifactRequest(t *testin }, } require.NoError(t, saveRunSummary(runOutputDir, summary, false)) + require.NoError(t, markArtifactDownloaded(runOutputDir, string(ArtifactSetAll))) result, ok := tryLoadCachedRunResult(context.Background(), WorkflowRun{DatabaseID: 123}, runOutputDir, concurrentRunDownloadParams{ evalsOnly: false, @@ -170,6 +171,7 @@ func TestTryLoadCachedRunResultUsesCacheWhenEvalsNotRequested(t *testing.T) { } require.NoError(t, saveRunSummary(runOutputDir, summary, false)) + require.NoError(t, markArtifactDownloaded(runOutputDir, string(ArtifactSetAll))) result, ok := tryLoadCachedRunResult(context.Background(), WorkflowRun{DatabaseID: 124}, runOutputDir, concurrentRunDownloadParams{ evalsOnly: false, @@ -221,6 +223,29 @@ func TestTryLoadCachedRunResultUsesCacheWhenRequestedArtifactsArePresent(t *test assert.True(t, result.Cached) } +func TestTryLoadCachedRunResultRequiresCompleteMarkerForAllArtifacts(t *testing.T) { + runOutputDir := t.TempDir() + summary := &RunSummary{ + CLIVersion: GetVersion(), + RunID: 127, + ProcessedAt: time.Now(), + Run: WorkflowRun{DatabaseID: 127}, + } + require.NoError(t, saveRunSummary(runOutputDir, summary, false)) + require.NoError(t, markArtifactDownloaded(runOutputDir, "activation")) + require.NoError(t, markArtifactDownloaded(runOutputDir, "usage")) + + result, ok := tryLoadCachedRunResult(context.Background(), WorkflowRun{DatabaseID: 127}, runOutputDir, concurrentRunDownloadParams{}) + assert.False(t, ok) + assert.Nil(t, result) + + require.NoError(t, markArtifactDownloaded(runOutputDir, string(ArtifactSetAll))) + result, ok = tryLoadCachedRunResult(context.Background(), WorkflowRun{DatabaseID: 127}, runOutputDir, concurrentRunDownloadParams{}) + require.True(t, ok) + require.NotNil(t, result) + assert.True(t, result.Cached) +} + // TestTryLoadCachedRunResultPersistsSafeItemsCountAfterBackfill verifies that when // tryLoadCachedRunResult heals a stale SafeItemsCount (0 → N) via backfillCacheHitIfNeeded, // the healed value is written back to run_summary.json on disk so downstream readers @@ -240,6 +265,7 @@ func TestTryLoadCachedRunResultPersistsSafeItemsCountAfterBackfill(t *testing.T) }, } require.NoError(t, saveRunSummary(runOutputDir, summary, false)) + require.NoError(t, markArtifactDownloaded(runOutputDir, string(ArtifactSetAll))) // Write a usage/activity/summary.json so backfill has something to pull from. activityPath := filepath.Join(runOutputDir, "usage", "activity", "summary.json") From 2462fe9a0cf8f13ef9e0cee829c0061d58216320 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 06:57:26 +0000 Subject: [PATCH 04/12] Preserve artifact cache markers Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs_download.go | 2 +- pkg/cli/logs_flatten_test.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/cli/logs_download.go b/pkg/cli/logs_download.go index 5078e9987c8..26560ca12b5 100644 --- a/pkg/cli/logs_download.go +++ b/pkg/cli/logs_download.go @@ -77,7 +77,7 @@ func flattenSingleFileArtifacts(outputDir string, verbose bool) error { } for _, entry := range entries { - if !entry.IsDir() { + if !entry.IsDir() || entry.Name() == downloadedArtifactsMarkerDir { continue } diff --git a/pkg/cli/logs_flatten_test.go b/pkg/cli/logs_flatten_test.go index e39c695cca0..fa932c81233 100644 --- a/pkg/cli/logs_flatten_test.go +++ b/pkg/cli/logs_flatten_test.go @@ -92,6 +92,14 @@ func TestFlattenSingleFileArtifacts(t *testing.T) { }, expectedDirs: []string{"empty-artifact"}, }, + { + name: "downloaded artifact markers are not flattened", + setup: func(dir string) error { + return markArtifactDownloaded(dir, string(ArtifactSetAll)) + }, + expectedDirs: []string{downloadedArtifactsMarkerDir}, + expectedFiles: []string{downloadedArtifactsMarkerDir + "/all"}, + }, { name: "regular files in output dir not affected", setup: func(dir string) error { From d09f267d86dedb04da60769c6156c06ad9876b22 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 07:18:21 +0000 Subject: [PATCH 05/12] Remove agentic logs cache TTL Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/ab-testing-advisor.lock.yml | 13 ------------- .github/workflows/agentic-logs-cache.yml | 18 ++---------------- .github/workflows/agentic-token-audit.lock.yml | 13 ------------- .../workflows/agentic-token-optimizer.lock.yml | 13 ------------- .../daily-ambient-context-optimizer.lock.yml | 13 ------------- .../workflows/daily-cli-tools-tester.lock.yml | 13 ------------- .github/workflows/daily-evals-report.lock.yml | 13 ------------- .../daily-safe-output-optimizer.lock.yml | 13 ------------- .../daily-security-observability.lock.yml | 13 ------------- .github/workflows/portfolio-analyst.lock.yml | 13 ------------- .../prompt-clustering-analysis.lock.yml | 13 ------------- .github/workflows/safe-output-health.lock.yml | 13 ------------- pkg/workflow/compiler_logs_cache.go | 15 --------------- pkg/workflow/compiler_logs_cache_test.go | 2 -- 14 files changed, 2 insertions(+), 176 deletions(-) diff --git a/.github/workflows/ab-testing-advisor.lock.yml b/.github/workflows/ab-testing-advisor.lock.yml index 16c2462419a..f422cca5c57 100644 --- a/.github/workflows/ab-testing-advisor.lock.yml +++ b/.github/workflows/ab-testing-advisor.lock.yml @@ -515,19 +515,6 @@ jobs: restore-keys: | agentic-logs- path: .github/aw/logs - - name: Enforce shared agentic logs cache TTL - shell: bash - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/agentic-logs-cache.yml b/.github/workflows/agentic-logs-cache.yml index 70734a01686..a6989682860 100644 --- a/.github/workflows/agentic-logs-cache.yml +++ b/.github/workflows/agentic-logs-cache.yml @@ -12,6 +12,7 @@ permissions: jobs: refresh: runs-on: ubuntu-latest + timeout-minutes: 50 steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -31,19 +32,6 @@ jobs: restore-keys: | agentic-logs- - - name: Enforce two-day cache TTL - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - - name: Build gh-aw run: make build @@ -55,12 +43,11 @@ jobs: mkdir -p .github/aw/logs ./gh-aw logs \ --start-date -30d \ - --count 1000 \ + --count 100000 \ --artifacts activation,usage \ --cache-before -30d \ --output .github/aw/logs \ --json > /tmp/agentic-logs-summary.json - date -u +%Y-%m-%dT%H:%M:%SZ > .github/aw/logs/.cache-refreshed-at - name: Save refreshed logs cache uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -77,7 +64,6 @@ jobs: echo "- Runs cached: $(jq -r '.runs | length' /tmp/agentic-logs-summary.json)" echo "- Artifacts: activation, usage" echo "- Window: 30 days" - echo "- Cache TTL: 2 days" else echo "- Cache refresh failed before producing a summary." fi diff --git a/.github/workflows/agentic-token-audit.lock.yml b/.github/workflows/agentic-token-audit.lock.yml index 854803c3991..ec46c9e2276 100644 --- a/.github/workflows/agentic-token-audit.lock.yml +++ b/.github/workflows/agentic-token-audit.lock.yml @@ -545,19 +545,6 @@ jobs: restore-keys: | agentic-logs- path: .github/aw/logs - - name: Enforce shared agentic logs cache TTL - shell: bash - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: diff --git a/.github/workflows/agentic-token-optimizer.lock.yml b/.github/workflows/agentic-token-optimizer.lock.yml index 5dd8a9cbbf0..9735f39f335 100644 --- a/.github/workflows/agentic-token-optimizer.lock.yml +++ b/.github/workflows/agentic-token-optimizer.lock.yml @@ -510,19 +510,6 @@ jobs: restore-keys: | agentic-logs- path: .github/aw/logs - - name: Enforce shared agentic logs cache TTL - shell: bash - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Download recent agentic workflow logs diff --git a/.github/workflows/daily-ambient-context-optimizer.lock.yml b/.github/workflows/daily-ambient-context-optimizer.lock.yml index 598b8631857..59763669fe8 100644 --- a/.github/workflows/daily-ambient-context-optimizer.lock.yml +++ b/.github/workflows/daily-ambient-context-optimizer.lock.yml @@ -553,19 +553,6 @@ jobs: restore-keys: | agentic-logs- path: .github/aw/logs - - name: Enforce shared agentic logs cache TTL - shell: bash - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: diff --git a/.github/workflows/daily-cli-tools-tester.lock.yml b/.github/workflows/daily-cli-tools-tester.lock.yml index ee09c4ff45f..a0ed41d5a9a 100644 --- a/.github/workflows/daily-cli-tools-tester.lock.yml +++ b/.github/workflows/daily-cli-tools-tester.lock.yml @@ -532,19 +532,6 @@ jobs: restore-keys: | agentic-logs- path: .github/aw/logs - - name: Enforce shared agentic logs cache TTL - shell: bash - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-evals-report.lock.yml b/.github/workflows/daily-evals-report.lock.yml index deec5b90bf3..63d75433115 100644 --- a/.github/workflows/daily-evals-report.lock.yml +++ b/.github/workflows/daily-evals-report.lock.yml @@ -538,19 +538,6 @@ jobs: restore-keys: | agentic-logs- path: .github/aw/logs - - name: Enforce shared agentic logs cache TTL - shell: bash - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/daily-safe-output-optimizer.lock.yml b/.github/workflows/daily-safe-output-optimizer.lock.yml index 87fbb667fc8..c95be5fbf76 100644 --- a/.github/workflows/daily-safe-output-optimizer.lock.yml +++ b/.github/workflows/daily-safe-output-optimizer.lock.yml @@ -609,19 +609,6 @@ jobs: restore-keys: | agentic-logs- path: .github/aw/logs - - name: Enforce shared agentic logs cache TTL - shell: bash - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Download logs from last 24 hours diff --git a/.github/workflows/daily-security-observability.lock.yml b/.github/workflows/daily-security-observability.lock.yml index c7354d5e484..71b06e6976a 100644 --- a/.github/workflows/daily-security-observability.lock.yml +++ b/.github/workflows/daily-security-observability.lock.yml @@ -577,19 +577,6 @@ jobs: restore-keys: | agentic-logs- path: .github/aw/logs - - name: Enforce shared agentic logs cache TTL - shell: bash - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - name: Setup Python environment run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - name: Install Python scientific libraries diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml index a992fc0e7db..541202ea134 100644 --- a/.github/workflows/portfolio-analyst.lock.yml +++ b/.github/workflows/portfolio-analyst.lock.yml @@ -526,19 +526,6 @@ jobs: restore-keys: | agentic-logs- path: .github/aw/logs - - name: Enforce shared agentic logs cache TTL - shell: bash - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - name: Setup Python environment run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" - name: Install Python scientific libraries diff --git a/.github/workflows/prompt-clustering-analysis.lock.yml b/.github/workflows/prompt-clustering-analysis.lock.yml index 1acd49f092b..a0c06be450e 100644 --- a/.github/workflows/prompt-clustering-analysis.lock.yml +++ b/.github/workflows/prompt-clustering-analysis.lock.yml @@ -578,19 +578,6 @@ jobs: restore-keys: | agentic-logs- path: .github/aw/logs - - name: Enforce shared agentic logs cache TTL - shell: bash - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - name: Install gh CLI run: | bash "${RUNNER_TEMP}/gh-aw/actions/install_gh_cli.sh" diff --git a/.github/workflows/safe-output-health.lock.yml b/.github/workflows/safe-output-health.lock.yml index df039cd2cb1..c77f6f90270 100644 --- a/.github/workflows/safe-output-health.lock.yml +++ b/.github/workflows/safe-output-health.lock.yml @@ -560,19 +560,6 @@ jobs: restore-keys: | agentic-logs- path: .github/aw/logs - - name: Enforce shared agentic logs cache TTL - shell: bash - run: | - marker=.github/aw/logs/.cache-refreshed-at - if [ -d .github/aw/logs ] && [ ! -f "$marker" ]; then - rm -rf .github/aw/logs - elif [ -f "$marker" ]; then - refreshed_at=$(cat "$marker") - if ! refreshed_epoch=$(date -d "$refreshed_at" +%s 2>/dev/null) \ - || [ "$refreshed_epoch" -lt "$(date -u -d '2 days ago' +%s)" ]; then - rm -rf .github/aw/logs - fi - fi - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} name: Download logs from last 24 hours diff --git a/pkg/workflow/compiler_logs_cache.go b/pkg/workflow/compiler_logs_cache.go index fbee5211c6e..e655c006022 100644 --- a/pkg/workflow/compiler_logs_cache.go +++ b/pkg/workflow/compiler_logs_cache.go @@ -34,21 +34,6 @@ func sharedLogsCacheRestoreSteps(data *WorkflowData) []GitHubActionStep { " agentic-logs-", " path: " + sharedLogsCachePath, }, - { - " - name: Enforce shared agentic logs cache TTL", - " shell: bash", - " run: |", - " marker=" + sharedLogsCachePath + "/.cache-refreshed-at", - " if [ -d " + sharedLogsCachePath + " ] && [ ! -f \"$marker\" ]; then", - " rm -rf " + sharedLogsCachePath, - " elif [ -f \"$marker\" ]; then", - " refreshed_at=$(cat \"$marker\")", - " if ! refreshed_epoch=$(date -d \"$refreshed_at\" +%s 2>/dev/null) \\", - " || [ \"$refreshed_epoch\" -lt \"$(date -u -d '2 days ago' +%s)\" ]; then", - " rm -rf " + sharedLogsCachePath, - " fi", - " fi", - }, } } diff --git a/pkg/workflow/compiler_logs_cache_test.go b/pkg/workflow/compiler_logs_cache_test.go index 2597b72acdb..b874a45ddbf 100644 --- a/pkg/workflow/compiler_logs_cache_test.go +++ b/pkg/workflow/compiler_logs_cache_test.go @@ -57,8 +57,6 @@ func TestGenerateSharedLogsCacheRestoreSteps(t *testing.T) { output := yaml.String() assert.Contains(t, output, "actions/cache/restore@") assert.Contains(t, output, "path: "+sharedLogsCachePath) - assert.Contains(t, output, "2 days ago") - assert.Contains(t, output, "[ ! -f \"$marker\" ]") assert.NotContains(t, output, "actions/cache/save@") } From 6e01fde4d16ee30ae4846d6bfee20df7140f6d3f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 08:04:00 +0000 Subject: [PATCH 06/12] Use single agentic logs cache key Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/ab-testing-advisor.lock.yml | 4 +--- .github/workflows/agentic-logs-cache.yml | 22 ++++++++++++++----- .../workflows/agentic-token-audit.lock.yml | 4 +--- .../agentic-token-optimizer.lock.yml | 4 +--- .../daily-ambient-context-optimizer.lock.yml | 4 +--- .../workflows/daily-cli-tools-tester.lock.yml | 4 +--- .github/workflows/daily-evals-report.lock.yml | 4 +--- .../daily-safe-output-optimizer.lock.yml | 4 +--- .../daily-security-observability.lock.yml | 4 +--- .github/workflows/portfolio-analyst.lock.yml | 4 +--- .../prompt-clustering-analysis.lock.yml | 4 +--- .github/workflows/safe-output-health.lock.yml | 4 +--- pkg/workflow/compiler_logs_cache.go | 9 ++++---- pkg/workflow/compiler_logs_cache_test.go | 2 ++ 14 files changed, 35 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ab-testing-advisor.lock.yml b/.github/workflows/ab-testing-advisor.lock.yml index f422cca5c57..be2a3c2c816 100644 --- a/.github/workflows/ab-testing-advisor.lock.yml +++ b/.github/workflows/ab-testing-advisor.lock.yml @@ -511,9 +511,7 @@ jobs: continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs path: .github/aw/logs - name: Configure Git credentials env: diff --git a/.github/workflows/agentic-logs-cache.yml b/.github/workflows/agentic-logs-cache.yml index a6989682860..8ac4ac6fee3 100644 --- a/.github/workflows/agentic-logs-cache.yml +++ b/.github/workflows/agentic-logs-cache.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: permissions: - actions: read + actions: write contents: read jobs: @@ -28,9 +28,7 @@ jobs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .github/aw/logs - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs - name: Build gh-aw run: make build @@ -49,11 +47,25 @@ jobs: --output .github/aw/logs \ --json > /tmp/agentic-logs-summary.json + - name: Delete previous logs cache + env: + GH_TOKEN: ${{ github.token }} + run: | + cache_id=$(gh cache list \ + --key agentic-logs \ + --ref "$GITHUB_REF" \ + --limit 1 \ + --json id,key \ + --jq '.[] | select(.key == "agentic-logs") | .id') + if [ -n "$cache_id" ]; then + gh cache delete "$cache_id" + fi + - name: Save refreshed logs cache uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .github/aw/logs - key: agentic-logs-${{ github.run_id }} + key: agentic-logs - name: Write summary if: always() diff --git a/.github/workflows/agentic-token-audit.lock.yml b/.github/workflows/agentic-token-audit.lock.yml index ec46c9e2276..606290373b9 100644 --- a/.github/workflows/agentic-token-audit.lock.yml +++ b/.github/workflows/agentic-token-audit.lock.yml @@ -541,9 +541,7 @@ jobs: continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs path: .github/aw/logs - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 diff --git a/.github/workflows/agentic-token-optimizer.lock.yml b/.github/workflows/agentic-token-optimizer.lock.yml index 9735f39f335..c8b3c9944bf 100644 --- a/.github/workflows/agentic-token-optimizer.lock.yml +++ b/.github/workflows/agentic-token-optimizer.lock.yml @@ -506,9 +506,7 @@ jobs: continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs path: .github/aw/logs - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/daily-ambient-context-optimizer.lock.yml b/.github/workflows/daily-ambient-context-optimizer.lock.yml index 59763669fe8..4ae7018d259 100644 --- a/.github/workflows/daily-ambient-context-optimizer.lock.yml +++ b/.github/workflows/daily-ambient-context-optimizer.lock.yml @@ -549,9 +549,7 @@ jobs: continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs path: .github/aw/logs - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 diff --git a/.github/workflows/daily-cli-tools-tester.lock.yml b/.github/workflows/daily-cli-tools-tester.lock.yml index a0ed41d5a9a..0fbe4fa30f0 100644 --- a/.github/workflows/daily-cli-tools-tester.lock.yml +++ b/.github/workflows/daily-cli-tools-tester.lock.yml @@ -528,9 +528,7 @@ jobs: continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs path: .github/aw/logs - name: Configure Git credentials env: diff --git a/.github/workflows/daily-evals-report.lock.yml b/.github/workflows/daily-evals-report.lock.yml index 63d75433115..7bb94e88db2 100644 --- a/.github/workflows/daily-evals-report.lock.yml +++ b/.github/workflows/daily-evals-report.lock.yml @@ -534,9 +534,7 @@ jobs: continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs path: .github/aw/logs - name: Configure Git credentials env: diff --git a/.github/workflows/daily-safe-output-optimizer.lock.yml b/.github/workflows/daily-safe-output-optimizer.lock.yml index c95be5fbf76..7dc5bd88be8 100644 --- a/.github/workflows/daily-safe-output-optimizer.lock.yml +++ b/.github/workflows/daily-safe-output-optimizer.lock.yml @@ -605,9 +605,7 @@ jobs: continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs path: .github/aw/logs - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/daily-security-observability.lock.yml b/.github/workflows/daily-security-observability.lock.yml index 71b06e6976a..241af198587 100644 --- a/.github/workflows/daily-security-observability.lock.yml +++ b/.github/workflows/daily-security-observability.lock.yml @@ -573,9 +573,7 @@ jobs: continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs path: .github/aw/logs - name: Setup Python environment run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml index 541202ea134..deb12b1b09d 100644 --- a/.github/workflows/portfolio-analyst.lock.yml +++ b/.github/workflows/portfolio-analyst.lock.yml @@ -522,9 +522,7 @@ jobs: continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs path: .github/aw/logs - name: Setup Python environment run: "# Create working directory for Python scripts\nmkdir -p /tmp/gh-aw/python\nmkdir -p /tmp/gh-aw/python/data\nmkdir -p /tmp/gh-aw/python/charts\nmkdir -p /tmp/gh-aw/python/artifacts\n\necho \"Python environment setup complete\"\necho \"Working directory: /tmp/gh-aw/python\"\necho \"Data directory: /tmp/gh-aw/python/data\"\necho \"Charts directory: /tmp/gh-aw/python/charts\"\necho \"Artifacts directory: /tmp/gh-aw/python/artifacts\"\n" diff --git a/.github/workflows/prompt-clustering-analysis.lock.yml b/.github/workflows/prompt-clustering-analysis.lock.yml index a0c06be450e..cd072860709 100644 --- a/.github/workflows/prompt-clustering-analysis.lock.yml +++ b/.github/workflows/prompt-clustering-analysis.lock.yml @@ -574,9 +574,7 @@ jobs: continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs path: .github/aw/logs - name: Install gh CLI run: | diff --git a/.github/workflows/safe-output-health.lock.yml b/.github/workflows/safe-output-health.lock.yml index c77f6f90270..03490cdab80 100644 --- a/.github/workflows/safe-output-health.lock.yml +++ b/.github/workflows/safe-output-health.lock.yml @@ -556,9 +556,7 @@ jobs: continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - key: agentic-logs-${{ github.run_id }} - restore-keys: | - agentic-logs- + key: agentic-logs path: .github/aw/logs - env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/pkg/workflow/compiler_logs_cache.go b/pkg/workflow/compiler_logs_cache.go index e655c006022..1173eca7aa3 100644 --- a/pkg/workflow/compiler_logs_cache.go +++ b/pkg/workflow/compiler_logs_cache.go @@ -2,7 +2,10 @@ package workflow import "strings" -const sharedLogsCachePath = ".github/aw/logs" +const ( + sharedLogsCacheKey = "agentic-logs" + sharedLogsCachePath = ".github/aw/logs" +) func usesSharedLogsCache(data *WorkflowData) bool { if !strings.Contains(data.On, "schedule") { @@ -29,9 +32,7 @@ func sharedLogsCacheRestoreSteps(data *WorkflowData) []GitHubActionStep { " continue-on-error: true", " uses: " + getCachedActionPin("actions/cache/restore", data), " with:", - " key: agentic-logs-${{ github.run_id }}", - " restore-keys: |", - " agentic-logs-", + " key: " + sharedLogsCacheKey, " path: " + sharedLogsCachePath, }, } diff --git a/pkg/workflow/compiler_logs_cache_test.go b/pkg/workflow/compiler_logs_cache_test.go index b874a45ddbf..823ca3985c0 100644 --- a/pkg/workflow/compiler_logs_cache_test.go +++ b/pkg/workflow/compiler_logs_cache_test.go @@ -56,7 +56,9 @@ func TestGenerateSharedLogsCacheRestoreSteps(t *testing.T) { output := yaml.String() assert.Contains(t, output, "actions/cache/restore@") + assert.Contains(t, output, "key: "+sharedLogsCacheKey) assert.Contains(t, output, "path: "+sharedLogsCachePath) + assert.NotContains(t, output, "restore-keys:") assert.NotContains(t, output, "actions/cache/save@") } From e197850eb2b90622dc6d1d3146665a02ed124065 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 08:25:27 +0000 Subject: [PATCH 07/12] Add draft ADR for shared agentic logs cache (PR #49082) Co-Authored-By: Claude Sonnet 4.6 --- ...ed-agentic-logs-for-scheduled-workflows.md | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 docs/adr/49082-cache-shared-agentic-logs-for-scheduled-workflows.md diff --git a/docs/adr/49082-cache-shared-agentic-logs-for-scheduled-workflows.md b/docs/adr/49082-cache-shared-agentic-logs-for-scheduled-workflows.md new file mode 100644 index 00000000000..fab27554ec8 --- /dev/null +++ b/docs/adr/49082-cache-shared-agentic-logs-for-scheduled-workflows.md @@ -0,0 +1,45 @@ +# ADR-49082: Cache Shared Agentic Logs for Scheduled Workflows + +**Date**: 2026-07-30 +**Status**: Draft +**Deciders**: Unknown + +--- + +### Context + +Scheduled log and audit workflows (e.g., `agentic-token-audit`, `daily-security-observability`, `safe-output-health`) each independently call `gh aw logs` or `gh aw audit`, triggering repeated downloads of the same `activation` and `usage` artifacts from the GitHub Actions API. These redundant transfers cause unnecessary API rate-limit pressure, increased network latency, and occasional timeouts for workflows that share the same 30-day data window. The `gh aw logs` downloader also needs a reliable way to distinguish a partial artifact download from a complete one when reusing cached data across runs; previously, the presence of a non-empty directory was used as a proxy for completeness, which was unreliable. + +### Decision + +We will introduce a single shared GitHub Actions cache (key: `agentic-logs`, path: `.github/aw/logs`) refreshed daily by a dedicated `agentic-logs-cache.yml` workflow. Consumer scheduled workflows restore this cache read-only before their `gh aw logs` / `gh aw audit` invocations. The Go downloader gains a `.downloaded-artifacts/` marker directory inside each run folder to record which artifact sets were successfully downloaded; cache-hit logic for unfiltered audit requests now requires a complete-download marker (`all`) rather than a non-empty directory. Consumer workflows that contain a custom checkout step restore the cache after that checkout to prevent cleanup from deleting cached data. + +### Alternatives Considered + +#### Alternative 1: Per-workflow independent download (status quo) + +Each scheduled workflow independently downloads the artifacts it needs from the GitHub API on every run. Simple and self-contained, but causes N redundant downloads per day (one per consuming workflow) for the same 30-day log window, contributing to rate-limit pressure and timeouts that motivated this change. + +#### Alternative 2: GitHub Actions artifact-based sharing + +Publish the pre-downloaded log directory as a named Actions artifact that consumer workflows download via `actions/download-artifact`. Artifacts are more visible in the Actions UI than cache entries. However, artifact retention defaults to 30–90 days with no built-in TTL enforcement at the key level; cache entries can be scoped to a ref and deleted/recreated atomically (as this PR does). Artifact-based sharing also doesn't integrate with the existing `actions/cache/restore` step pattern already used elsewhere in these workflows. + +### Consequences + +#### Positive +- Eliminates redundant per-workflow artifact downloads: all consumers share a single 30-day log snapshot refreshed once daily. +- Adds a reliable artifact-presence marker system (`markArtifactDownloaded`) in the Go downloader, fixing the existing fragility where a non-empty directory was treated as a complete download. +- The `shouldDownloadWorkflowRunLogs` predicate generalises the previous `isUsageOnlyArtifactFilter` check, correctly skipping heavyweight workflow-run log downloads for any activation/usage-only filter, not just the single-artifact case. + +#### Negative +- Adds a new daily maintenance workflow (`agentic-logs-cache.yml`) that must succeed for consumers to benefit; if it fails, consumers fall back to live downloads (via `continue-on-error: true`), but the operational surface area increases. +- Cache entries are scoped to `GITHUB_REF`; feature branches will not have a warm cache and will always fall through to live downloads. +- The marker-based completeness check (`ArtifactSetAll` marker required for unfiltered requests) is a breaking change in cache-hit semantics: existing run directories lacking the marker will be re-downloaded on next access. + +#### Neutral +- Consumer workflows require a `actions/cache/restore` step injected by the compiler; the compiler (`compiler_logs_cache.go`) detects eligibility by checking for `schedule:` in `on:` and `gh aw logs` / `gh aw audit` in workflow content. +- The shared cache path (`.github/aw/logs`) is a repository-relative directory, which means checkout cleanup will delete it unless the restore step follows the custom checkout — the compiler handles this ordering automatically. + +--- + +*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.* From 0b4022ae79e3ebea3a012978eec8e32f32d2ec22 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 08:51:41 +0000 Subject: [PATCH 08/12] Fix marker-dir loop, all-marker shortcut, comment detection, and retry path markers Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../workflows/daily-cli-tools-tester.lock.yml | 7 ------- pkg/cli/logs_artifact_set.go | 17 ++++++++++++----- pkg/cli/logs_artifact_set_test.go | 12 ++++++++++++ pkg/cli/logs_download.go | 9 ++++++++- pkg/workflow/compiler_logs_cache.go | 9 +++++++-- pkg/workflow/compiler_logs_cache_test.go | 10 ++++++++++ 6 files changed, 49 insertions(+), 15 deletions(-) diff --git a/.github/workflows/daily-cli-tools-tester.lock.yml b/.github/workflows/daily-cli-tools-tester.lock.yml index 0fbe4fa30f0..1ceaa25b398 100644 --- a/.github/workflows/daily-cli-tools-tester.lock.yml +++ b/.github/workflows/daily-cli-tools-tester.lock.yml @@ -523,13 +523,6 @@ jobs: with: name: activation path: /tmp/gh-aw - - name: Restore shared agentic logs cache - id: restore-agentic-logs-cache - continue-on-error: true - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - key: agentic-logs - path: .github/aw/logs - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/pkg/cli/logs_artifact_set.go b/pkg/cli/logs_artifact_set.go index f4b4fe7baeb..94ef923b233 100644 --- a/pkg/cli/logs_artifact_set.go +++ b/pkg/cli/logs_artifact_set.go @@ -257,15 +257,22 @@ func findMissingFilterEntries(filter []string, outputDir string) []string { if e.IsDir() { dirs = append(dirs, e.Name()) } - if markers, markerErr := os.ReadDir(filepath.Join(outputDir, downloadedArtifactsMarkerDir)); markerErr == nil { - for _, marker := range markers { - if !marker.IsDir() { - dirs = append(dirs, marker.Name()) - } + } + if markers, markerErr := os.ReadDir(filepath.Join(outputDir, downloadedArtifactsMarkerDir)); markerErr == nil { + for _, marker := range markers { + if !marker.IsDir() { + dirs = append(dirs, marker.Name()) } } } + // A complete-download marker satisfies every filtered request: if it is + // present the caller already downloaded all artifacts for this run. + if slices.Contains(dirs, string(ArtifactSetAll)) { + artifactSetLog.Printf("Complete-download marker present in %s; all filter entries satisfied", outputDir) + return nil + } + var missing []string for _, f := range filter { found := false diff --git a/pkg/cli/logs_artifact_set_test.go b/pkg/cli/logs_artifact_set_test.go index a80ebf34e3e..a40115afeac 100644 --- a/pkg/cli/logs_artifact_set_test.go +++ b/pkg/cli/logs_artifact_set_test.go @@ -440,6 +440,18 @@ func TestFindMissingFilterEntriesUsesDownloadedMarkers(t *testing.T) { assert.Equal(t, []string{"agent"}, findMissingFilterEntries([]string{"activation", "agent"}, dir)) } +func TestFindMissingFilterEntriesAllMarkerSatisfiesFiltered(t *testing.T) { + // A complete-download marker (ArtifactSetAll) should satisfy every filtered + // request even when individual artifact directories no longer exist (e.g. after + // flattenSingleFileArtifacts removes them). + dir := t.TempDir() + require.NoError(t, markArtifactDownloaded(dir, string(ArtifactSetAll))) + + assert.Nil(t, findMissingFilterEntries([]string{"activation"}, dir)) + assert.Nil(t, findMissingFilterEntries([]string{"activation", "usage"}, dir)) + assert.Nil(t, findMissingFilterEntries([]string{string(ArtifactSetAll)}, dir)) +} + func TestMarkArtifactDownloadedRejectsInvalidNames(t *testing.T) { err := markArtifactDownloaded(t.TempDir(), "../activation") require.Error(t, err) diff --git a/pkg/cli/logs_download.go b/pkg/cli/logs_download.go index 26560ca12b5..36ef057350e 100644 --- a/pkg/cli/logs_download.go +++ b/pkg/cli/logs_download.go @@ -684,6 +684,9 @@ func retryCriticalArtifacts(ctx context.Context, opts downloadArtifactsOptions) } } else { logsDownloadLog.Printf("Successfully downloaded artifact %q individually", name) + // Marker write failures are non-fatal in the retry path: retryCriticalArtifacts + // is a best-effort recovery after a partial bulk download, so a missing marker + // only causes a redundant re-download on the next run (not data loss). if err := markArtifactDownloaded(opts.outputDir, name); err != nil { logsDownloadLog.Printf("Failed to mark artifact %q as downloaded: %v", name, err) } @@ -934,7 +937,11 @@ func downloadRunArtifacts(ctx context.Context, opts downloadArtifactsOptions) er // workflow logs rather than artifact content. return ErrNoArtifacts } - if err == nil { + // Write the complete-download marker when the bulk download succeeded with no + // errors, or when some non-zip artifacts were skipped but critical artifacts were + // recovered via retryCriticalArtifacts. In the skipped-non-zip case the directory + // is non-empty (guarded above), so marking prevents redundant re-downloads. + if err == nil || skippedNonZipArtifacts { if markerErr := markArtifactDownloaded(opts.outputDir, string(ArtifactSetAll)); markerErr != nil { return markerErr } diff --git a/pkg/workflow/compiler_logs_cache.go b/pkg/workflow/compiler_logs_cache.go index 1173eca7aa3..6b5fff0caec 100644 --- a/pkg/workflow/compiler_logs_cache.go +++ b/pkg/workflow/compiler_logs_cache.go @@ -13,8 +13,13 @@ func usesSharedLogsCache(data *WorkflowData) bool { } content := data.CustomSteps + "\n" + data.MarkdownContent for _, command := range []string{"gh aw logs", "./gh-aw logs", "gh aw audit", "./gh-aw audit"} { - if strings.Contains(content, command) { - return true + for line := range strings.SplitSeq(content, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + if strings.Contains(line, command) { + return true + } } } return false diff --git a/pkg/workflow/compiler_logs_cache_test.go b/pkg/workflow/compiler_logs_cache_test.go index 823ca3985c0..ccbe53096c6 100644 --- a/pkg/workflow/compiler_logs_cache_test.go +++ b/pkg/workflow/compiler_logs_cache_test.go @@ -33,6 +33,16 @@ func TestUsesSharedLogsCache(t *testing.T) { data: WorkflowData{On: "schedule: daily", MarkdownContent: "Review issues."}, want: false, }, + { + name: "command in YAML comment is ignored", + data: WorkflowData{On: "schedule: daily", CustomSteps: "# run: gh aw logs --json"}, + want: false, + }, + { + name: "command in shell comment is ignored", + data: WorkflowData{On: "schedule: daily", CustomSteps: " # do not run: gh aw audit"}, + want: false, + }, } for _, tt := range tests { From d13fe960ba2143bd3a11886a3db24d8dd103d0da Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:22:30 +0000 Subject: [PATCH 09/12] Fix multi-checkout cache restore ordering, case-collision all-marker, and add build tag Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- pkg/cli/logs_download.go | 10 ++-- pkg/workflow/compiler_logs_cache_test.go | 44 +++++++++++++- pkg/workflow/compiler_workflow_helpers.go | 44 ++++++++++++++ pkg/workflow/compiler_yaml_main_job_test.go | 2 +- pkg/workflow/compiler_yaml_runtime_setup.go | 63 ++++++++++++++------- 5 files changed, 137 insertions(+), 26 deletions(-) diff --git a/pkg/cli/logs_download.go b/pkg/cli/logs_download.go index 36ef057350e..1b2d24fd537 100644 --- a/pkg/cli/logs_download.go +++ b/pkg/cli/logs_download.go @@ -938,10 +938,12 @@ func downloadRunArtifacts(ctx context.Context, opts downloadArtifactsOptions) er return ErrNoArtifacts } // Write the complete-download marker when the bulk download succeeded with no - // errors, or when some non-zip artifacts were skipped but critical artifacts were - // recovered via retryCriticalArtifacts. In the skipped-non-zip case the directory - // is non-empty (guarded above), so marking prevents redundant re-downloads. - if err == nil || skippedNonZipArtifacts { + // errors, when some non-zip artifacts were skipped but critical artifacts were + // recovered via retryCriticalArtifacts, or when a case-collision caused the bulk + // download to abort but all artifacts were successfully retried individually. + // In all three cases the directory is non-empty (guarded above for non-zip), + // so marking prevents an unbounded re-download loop on subsequent runs. + if err == nil || skippedNonZipArtifacts || skippedCaseCollisionArtifacts { if markerErr := markArtifactDownloaded(opts.outputDir, string(ArtifactSetAll)); markerErr != nil { return markerErr } diff --git a/pkg/workflow/compiler_logs_cache_test.go b/pkg/workflow/compiler_logs_cache_test.go index ccbe53096c6..142ae135910 100644 --- a/pkg/workflow/compiler_logs_cache_test.go +++ b/pkg/workflow/compiler_logs_cache_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package workflow import ( @@ -83,7 +85,7 @@ func TestSharedLogsCacheRestoreFollowsCustomCheckout(t *testing.T) { var yaml strings.Builder compiler := &Compiler{} - compiler.addCustomStepsWithRuntimeInsertion(&yaml, data.CustomSteps, sharedLogsCacheRestoreSteps(data), nil, false) + compiler.addCustomStepsWithRuntimeInsertion(&yaml, data.CustomSteps, nil, sharedLogsCacheRestoreSteps(data), nil, false) output := yaml.String() checkoutIndex := strings.Index(output, "uses: actions/checkout@v4") @@ -92,3 +94,43 @@ func TestSharedLogsCacheRestoreFollowsCustomCheckout(t *testing.T) { assert.Greater(t, restoreIndex, checkoutIndex) assert.Greater(t, logsIndex, restoreIndex) } + +// TestSharedLogsCacheRestoreFollowsLastCheckoutInMultiCheckout verifies that when custom steps +// contain multiple checkout actions (a multi-checkout scenario), the cache restore step is +// inserted after the LAST checkout, not the first. This ensures a later root checkout cannot +// wipe .github/aw/logs before the logs/audit command uses the cached data. +func TestSharedLogsCacheRestoreFollowsLastCheckoutInMultiCheckout(t *testing.T) { + cache := NewActionCache(t.TempDir()) + data := &WorkflowData{ + On: "schedule: daily", + CustomSteps: "steps:\n" + + " - uses: actions/checkout@v4\n" + + " with:\n" + + " repository: org/repo-a\n" + + " - name: Setup step\n" + + " run: echo setup\n" + + " - uses: actions/checkout@v4\n" + + " with:\n" + + " repository: org/repo-b\n" + + " - name: Download logs\n" + + " run: gh aw logs", + ActionCache: cache, + ActionResolver: NewActionResolver(cache), + } + var yaml strings.Builder + compiler := &Compiler{} + + compiler.addCustomStepsWithRuntimeInsertion(&yaml, data.CustomSteps, nil, sharedLogsCacheRestoreSteps(data), nil, false) + + output := yaml.String() + firstCheckoutIndex := strings.Index(output, "uses: actions/checkout@v4") + lastCheckoutIndex := strings.LastIndex(output, "uses: actions/checkout@v4") + restoreIndex := strings.Index(output, "Restore shared agentic logs cache") + logsIndex := strings.Index(output, "run: gh aw logs") + + // Cache restore must appear after the LAST checkout, not after the first. + assert.Greater(t, restoreIndex, lastCheckoutIndex, "cache restore should follow the last checkout") + assert.Greater(t, logsIndex, restoreIndex, "gh aw logs should follow the cache restore") + // Verify there are indeed two separate checkouts. + assert.NotEqual(t, firstCheckoutIndex, lastCheckoutIndex, "should have two distinct checkout positions") +} diff --git a/pkg/workflow/compiler_workflow_helpers.go b/pkg/workflow/compiler_workflow_helpers.go index f288ba93f1c..6b3958dbc17 100644 --- a/pkg/workflow/compiler_workflow_helpers.go +++ b/pkg/workflow/compiler_workflow_helpers.go @@ -17,6 +17,50 @@ func ContainsCheckout(customSteps string) bool { return found } +// findLastCheckoutStepIndex returns the zero-based index of the last +// actions/checkout step in customSteps and true, or (0, false) if none is found. +// It uses the same parsing strategy as findFirstCheckoutStepIndex. +func findLastCheckoutStepIndex(customSteps string) (int, bool) { + if customSteps == "" { + return 0, false + } + lastIndex := -1 + found := false + + // Try the wrapped form first: "steps:\n - ...\n" + var wrapper struct { + Steps []map[string]any `yaml:"steps"` + } + if err := yaml.Unmarshal([]byte(customSteps), &wrapper); err == nil { + if len(wrapper.Steps) > 0 { + for i, step := range wrapper.Steps { + uses, ok := step["uses"].(string) + if ok && isCheckoutActionReference(uses) { + lastIndex = i + found = true + } + } + return lastIndex, found + } + } + + // Fall back to the bare sequence form: "- uses: ...\n" + var steps []map[string]any + if err := yaml.Unmarshal([]byte(customSteps), &steps); err != nil { + // Malformed YAML: we cannot safely determine a checkout step index. + return 0, false + } + + for i, step := range steps { + uses, ok := step["uses"].(string) + if ok && isCheckoutActionReference(uses) { + lastIndex = i + found = true + } + } + return lastIndex, found +} + // findFirstCheckoutStepIndex returns the zero-based index of the first // actions/checkout step in customSteps and true, or (0, false) if none is found. // diff --git a/pkg/workflow/compiler_yaml_main_job_test.go b/pkg/workflow/compiler_yaml_main_job_test.go index 810ac013ebe..c8b30cab5b8 100644 --- a/pkg/workflow/compiler_yaml_main_job_test.go +++ b/pkg/workflow/compiler_yaml_main_job_test.go @@ -505,7 +505,7 @@ func TestAddCustomStepsWithRuntimeInsertion(t *testing.T) { compiler := NewCompiler() var yaml strings.Builder - compiler.addCustomStepsWithRuntimeInsertion(&yaml, tt.customSteps, tt.runtimeSetupSteps, tt.tools, tt.ensureArcNodePath) + compiler.addCustomStepsWithRuntimeInsertion(&yaml, tt.customSteps, tt.runtimeSetupSteps, nil, tt.tools, tt.ensureArcNodePath) result := yaml.String() for _, expected := range tt.expectInOutput { diff --git a/pkg/workflow/compiler_yaml_runtime_setup.go b/pkg/workflow/compiler_yaml_runtime_setup.go index e2b0a7abb60..702f740025f 100644 --- a/pkg/workflow/compiler_yaml_runtime_setup.go +++ b/pkg/workflow/compiler_yaml_runtime_setup.go @@ -21,9 +21,6 @@ import ( // themselves contain a checkout action (used by the caller to compute needsGitConfig). func (c *Compiler) generateRuntimeAndWorkspaceSetupSteps(yaml *strings.Builder, data *WorkflowData, needsCheckout bool) bool { runtimeSetupSteps, customStepsContainCheckout := c.prepareRuntimeSetupAndCheckoutInfo(data) - if customStepsContainCheckout { - runtimeSetupSteps = append(runtimeSetupSteps, sharedLogsCacheRestoreSteps(data)...) - } compilerYamlLog.Printf("Custom steps contain checkout: %t (len(customSteps)=%d)", customStepsContainCheckout, len(data.CustomSteps)) c.emitRuntimeSetupPrelude(yaml, data, needsCheckout, customStepsContainCheckout, runtimeSetupSteps) @@ -202,13 +199,16 @@ func (c *Compiler) emitCustomSteps(yaml *strings.Builder, data *WorkflowData, cu if hasDIFCProxyNeeded(data) { customStepsToEmit = injectProxyEnvIntoCustomSteps(customStepsToEmit) } - if customStepsContainCheckout && len(runtimeSetupSteps) > 0 { - // Custom steps contain checkout and we have runtime steps to insert - // Insert runtime steps after the first checkout step - compilerYamlLog.Printf("Calling addCustomStepsWithRuntimeInsertion: %d runtime steps to insert after checkout", len(runtimeSetupSteps)) - c.addCustomStepsWithRuntimeInsertion(yaml, customStepsToEmit, runtimeSetupSteps, data.ParsedTools, isArcDindTopology(data)) + if customStepsContainCheckout && (len(runtimeSetupSteps) > 0 || len(sharedLogsCacheRestoreSteps(data)) > 0) { + // Custom steps contain checkout: insert runtime steps after the first checkout and + // the cache restore after the last checkout. Inserting the cache restore after the + // last checkout ensures that a multi-checkout custom steps block (where a later root + // checkout would wipe .github/aw/logs) does not leave the consumer without cached data. + cacheRestoreSteps := sharedLogsCacheRestoreSteps(data) + compilerYamlLog.Printf("Calling addCustomStepsWithRuntimeInsertion: %d runtime steps after first checkout, %d cache-restore steps after last checkout", len(runtimeSetupSteps), len(cacheRestoreSteps)) + c.addCustomStepsWithRuntimeInsertion(yaml, customStepsToEmit, runtimeSetupSteps, cacheRestoreSteps, data.ParsedTools, isArcDindTopology(data)) } else { - // No checkout in custom steps or no runtime steps, just add custom steps as-is + // No checkout in custom steps or no steps to insert, just add custom steps as-is compilerYamlLog.Printf("Calling addCustomStepsAsIs (customStepsContainCheckout=%t, runtimeStepsCount=%d)", customStepsContainCheckout, len(runtimeSetupSteps)) c.addCustomStepsAsIs(yaml, customStepsToEmit) } @@ -344,11 +344,18 @@ func (c *Compiler) addCustomStepsAsIs(yaml *strings.Builder, customSteps string) } } -// addCustomStepsWithRuntimeInsertion adds custom steps and inserts runtime steps after the first checkout. +// addCustomStepsWithRuntimeInsertion adds custom steps and inserts runtime steps after the first +// checkout and postLastCheckoutSteps (e.g. cache restore) after the last checkout. In the common +// single-checkout case both insertions happen at the same point. For multi-checkout workflows +// (where a later root checkout would wipe a previously restored path such as .github/aw/logs) +// the post-last-checkout steps are deferred until after the final checkout so the path remains +// intact when the subsequent command runs. // Like addCustomStepsAsIs it sanitizes any ${{ ... }} expressions found in run: fields before writing. -func (c *Compiler) addCustomStepsWithRuntimeInsertion(yaml *strings.Builder, customSteps string, runtimeSetupSteps []GitHubActionStep, tools *ToolsConfig, ensureArcDindNodePath bool) { +func (c *Compiler) addCustomStepsWithRuntimeInsertion(yaml *strings.Builder, customSteps string, runtimeSetupSteps []GitHubActionStep, postLastCheckoutSteps []GitHubActionStep, tools *ToolsConfig, ensureArcDindNodePath bool) { customSteps = c.sanitizeAndWarnCustomSteps(customSteps) - checkoutStepIndex, hasCheckoutStep := findFirstCheckoutStepIndex(customSteps) + firstCheckoutIndex, hasCheckoutStep := findFirstCheckoutStepIndex(customSteps) + lastCheckoutIndex, _ := findLastCheckoutStepIndex(customSteps) + hasPostLastSteps := len(postLastCheckoutSteps) > 0 // Remove "steps:" line and adjust indentation lines := strings.Split(customSteps, "\n") if len(lines) <= 1 { @@ -356,6 +363,7 @@ func (c *Compiler) addCustomStepsWithRuntimeInsertion(yaml *strings.Builder, cus } insertedRuntime := false + insertedPostLast := false i := 1 // Start from index 1 to skip "steps:" line currentStepIndex := -1 stepIndent := -1 @@ -386,12 +394,13 @@ func (c *Compiler) addCustomStepsWithRuntimeInsertion(yaml *strings.Builder, cus isStepStart = indent == stepIndent } - if isStepStart && !insertedRuntime { + if isStepStart { currentStepIndex++ - isCheckoutStep := hasCheckoutStep && currentStepIndex == checkoutStepIndex + isFirstCheckout := hasCheckoutStep && currentStepIndex == firstCheckoutIndex && !insertedRuntime + isLastCheckout := hasPostLastSteps && hasCheckoutStep && currentStepIndex == lastCheckoutIndex && !insertedPostLast - if isCheckoutStep { - // This is a checkout step, copy all its lines until the next step + if isFirstCheckout || isLastCheckout { + // This is a checkout step (first, last, or both): copy all its lines until the next step i++ for i < len(lines) { nextLine := lines[i] @@ -411,11 +420,25 @@ func (c *Compiler) addCustomStepsWithRuntimeInsertion(yaml *strings.Builder, cus i++ } - // Now insert runtime steps after the checkout step - compilerYamlLog.Printf("Inserting %d runtime setup steps after checkout in custom steps", len(runtimeSetupSteps)) - c.emitRuntimeSetupSteps(yaml, runtimeSetupSteps, ensureArcDindNodePath) + if isFirstCheckout { + // Insert runtime steps after the first checkout step + compilerYamlLog.Printf("Inserting %d runtime setup steps after first checkout in custom steps", len(runtimeSetupSteps)) + c.emitRuntimeSetupSteps(yaml, runtimeSetupSteps, ensureArcDindNodePath) + insertedRuntime = true + } + if isLastCheckout || (isFirstCheckout && firstCheckoutIndex == lastCheckoutIndex) { + // Insert post-last-checkout steps (e.g. cache restore) after the last checkout. + // When first == last (single-checkout), this runs immediately after runtime insertion above. + compilerYamlLog.Printf("Inserting %d post-last-checkout steps after last checkout in custom steps", len(postLastCheckoutSteps)) + for _, step := range postLastCheckoutSteps { + for _, stepLine := range step { + yaml.WriteString(stepLine) + yaml.WriteByte('\n') + } + } + insertedPostLast = true + } - insertedRuntime = true continue // Continue with the next iteration (i is already advanced) } } From 73d54a0dd806351ee10fb3d7a995d45209437bfa Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:44:28 +0000 Subject: [PATCH 10/12] Improve incremental artifact download: skip already-present data, minimize API calls Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs_artifact_set_test.go | 34 +++++++++++++++ pkg/cli/logs_download.go | 70 +++++++++++++++++++++++++------ 2 files changed, 92 insertions(+), 12 deletions(-) diff --git a/pkg/cli/logs_artifact_set_test.go b/pkg/cli/logs_artifact_set_test.go index a40115afeac..ce86e1a7197 100644 --- a/pkg/cli/logs_artifact_set_test.go +++ b/pkg/cli/logs_artifact_set_test.go @@ -457,3 +457,37 @@ func TestMarkArtifactDownloadedRejectsInvalidNames(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "invalid artifact name") } + +// TestFindMissingFilterEntriesIncrementalScenario validates the key scenario used by +// the incremental unfiltered download: a previous filtered pass wrote per-artifact +// markers with the full API artifact name (e.g. "abc123-activation"), and the +// subsequent unfiltered pass supplies the same full names to findMissingFilterEntries +// to determine which are still missing. +func TestFindMissingFilterEntriesIncrementalScenario(t *testing.T) { + dir := t.TempDir() + + // Simulate a previous filtered download that wrote markers with full API artifact names. + require.NoError(t, markArtifactDownloaded(dir, "abc123-activation")) + require.NoError(t, markArtifactDownloaded(dir, "abc123-usage")) + + // The unfiltered incremental check passes full API names as the filter. + // activation and usage are found via their exact-match markers; agent is missing. + result := findMissingFilterEntries([]string{"abc123-activation", "abc123-usage", "abc123-agent"}, dir) + assert.Equal(t, []string{"abc123-agent"}, result) + + // After downloading agent (marker written), nothing is missing. + require.NoError(t, markArtifactDownloaded(dir, "abc123-agent")) + assert.Nil(t, findMissingFilterEntries([]string{"abc123-activation", "abc123-usage", "abc123-agent"}, dir)) +} + +// TestFindMissingFilterEntriesAllMarkerSatisfiesFullNames verifies that the +// complete-download marker satisfies a filter containing full API artifact names +// (as used by the incremental unfiltered download check). +func TestFindMissingFilterEntriesAllMarkerSatisfiesFullNames(t *testing.T) { + dir := t.TempDir() + require.NoError(t, markArtifactDownloaded(dir, string(ArtifactSetAll))) + + // Full API artifact names — all satisfied by the 'all' marker. + assert.Nil(t, findMissingFilterEntries([]string{"abc123-activation", "abc123-usage", "abc123-agent"}, dir)) + assert.Nil(t, findMissingFilterEntries([]string{"activation", "usage", "agent"}, dir)) +} diff --git a/pkg/cli/logs_download.go b/pkg/cli/logs_download.go index 1b2d24fd537..cb7ebc16c20 100644 --- a/pkg/cli/logs_download.go +++ b/pkg/cli/logs_download.go @@ -727,17 +727,15 @@ func downloadRunArtifacts(ctx context.Context, opts downloadArtifactsOptions) er opts.artifactFilter = missing // Fall through to the download code below (MkdirAll is a no-op for existing dir). } else { - // No filter — caller wants all artifacts. Keep the existing behaviour: - // only a complete bulk download marker can satisfy an all-artifacts request. + // No filter — caller wants all artifacts. The complete-download marker is + // sufficient to skip the download; no cached summary is required because + // the marker itself guarantees all artifact data is present on disk. if len(findMissingFilterEntries([]string{string(ArtifactSetAll)}, opts.outputDir)) == 0 { - if summary, ok := loadRunSummary(opts.outputDir, opts.verbose); ok { - // Valid cached summary exists, skip download - logsDownloadLog.Printf("Using cached artifacts for run %d", opts.runID) - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Using cached artifacts for run %d at %s (from %s)", opts.runID, opts.outputDir, summary.ProcessedAt.Format("2006-01-02 15:04:05")))) - } - return nil + logsDownloadLog.Printf("Using cached artifacts for run %d", opts.runID) + if shouldLogProgress { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("All artifacts already present for run %d, skipping download", opts.runID))) } + return nil } if opts.verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Run folder for %d is missing the complete artifact marker; downloading all artifacts", opts.runID))) @@ -783,15 +781,51 @@ func downloadRunArtifacts(ctx context.Context, opts downloadArtifactsOptions) er } } + // Incremental download: when the output directory already holds some artifacts but + // lacks the complete-download marker, check which artifacts are actually missing and + // restrict the download to those. This avoids re-fetching data that was already + // transferred during a previous filtered pass (e.g. activation+usage) even when the + // caller now requests the full artifact set. + var incrementalUnfilteredDownload bool + if listErr == nil && len(downloadableNames) > 0 && len(opts.artifactFilter) == 0 && + fileutil.DirExists(opts.outputDir) && !fileutil.IsDirEmpty(opts.outputDir) { + missingNames := findMissingFilterEntries(downloadableNames, opts.outputDir) + if len(missingNames) == 0 { + // All artifacts are already on disk. Confirm with the complete-download marker + // so that future unfiltered requests benefit from the fast-path check above. + logsDownloadLog.Printf("All %d artifacts already present for run %d (incremental check)", len(downloadableNames), opts.runID) + if shouldLogProgress { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("All artifacts already present for run %d, skipping download", opts.runID))) + } + if markerErr := markArtifactDownloaded(opts.outputDir, string(ArtifactSetAll)); markerErr != nil { + return markerErr + } + return nil + } + if len(missingNames) < len(downloadableNames) { + // Some artifacts are already present; narrow the download to the remainder. + logsDownloadLog.Printf("Incremental download for run %d: %d/%d artifacts missing: %v", + opts.runID, len(missingNames), len(downloadableNames), missingNames) + if shouldLogProgress { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf( + "Incremental download for run %d: fetching %d missing artifact(s): %v", + opts.runID, len(missingNames), missingNames))) + } + downloadableNames = missingNames + incrementalUnfilteredDownload = true + } + } + // Start spinner for network operation spinner := console.NewSpinner(fmt.Sprintf("Downloading artifacts for run %d...", opts.runID)) if !opts.verbose { spinner.Start() } - if len(dockerBuildArtifacts) > 0 || len(opts.artifactFilter) > 0 { - // When .dockerbuild artifacts are present or an artifact filter is active, download - // only the selected artifacts individually instead of using the bulk downloader. + if len(dockerBuildArtifacts) > 0 || len(opts.artifactFilter) > 0 || incrementalUnfilteredDownload { + // When .dockerbuild artifacts are present, an artifact filter is active, or an + // incremental top-up is needed, download only the selected artifacts individually + // instead of using the bulk downloader. // The bulk downloader (gh run download without --name) cannot apply a name filter, // and it aborts on non-zip artifacts. if !opts.verbose { @@ -822,6 +856,18 @@ func downloadRunArtifacts(ctx context.Context, opts downloadArtifactsOptions) er // Downloads were attempted but none succeeded; treat as no artifacts. return ErrNoArtifacts } + // Write the complete-download marker when this was an unfiltered request routed + // through the individual download path (dockerbuild artifacts present, or an + // incremental top-up of an existing run directory). Only mark as complete when + // all expected artifacts are actually present, so that partially-failed downloads + // are retried next time rather than treated as complete. + if len(opts.artifactFilter) == 0 && len(downloadableNames) > 0 { + if len(findMissingFilterEntries(downloadableNames, opts.outputDir)) == 0 { + if markerErr := markArtifactDownloaded(opts.outputDir, string(ArtifactSetAll)); markerErr != nil { + return markerErr + } + } + } } else { // No .dockerbuild artifacts detected (or listing failed) — use efficient bulk download. // Build gh run download command with optional repo/hostname override for cross-repo and multi-host support From f1ecd4621e633e513f9c1b5dafb521780ba522ae Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 10:01:28 +0000 Subject: [PATCH 11/12] pr-finisher: no new changes - reviewing unresolved threads Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- .github/skills/agentic-workflows/SKILL.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/skills/agentic-workflows/SKILL.md b/.github/skills/agentic-workflows/SKILL.md index 6f24708a24e..6fb19019416 100644 --- a/.github/skills/agentic-workflows/SKILL.md +++ b/.github/skills/agentic-workflows/SKILL.md @@ -35,6 +35,7 @@ Load these files from `github/gh-aw` (they are not available locally). - `.github/aw/evals.md` - `.github/aw/experiments.md` - `.github/aw/github-agentic-workflows.md` +- `.github/aw/github-mcp-server-pagination.md` - `.github/aw/github-mcp-server.md` - `.github/aw/instructions.md` - `.github/aw/linter-workflows.md` @@ -64,10 +65,12 @@ Load these files from `github/gh-aw` (they are not available locally). - `.github/aw/subagents.md` - `.github/aw/syntax-agentic.md` - `.github/aw/syntax-core.md` +- `.github/aw/syntax-engine.md` - `.github/aw/syntax-tools-imports.md` - `.github/aw/syntax.md` - `.github/aw/test-coverage.md` - `.github/aw/test-expression.md` +- `.github/aw/token-optimization-caching-budgets.md` - `.github/aw/token-optimization.md` - `.github/aw/triggers.md` - `.github/aw/update-agentic-workflow.md` From 9c1ba82e0b4f796b52e70fb276b90ad9d690ae33 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 10:06:03 +0000 Subject: [PATCH 12/12] Apply remaining changes Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- .github/skills/agentic-workflows/SKILL.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/skills/agentic-workflows/SKILL.md b/.github/skills/agentic-workflows/SKILL.md index 6fb19019416..6f24708a24e 100644 --- a/.github/skills/agentic-workflows/SKILL.md +++ b/.github/skills/agentic-workflows/SKILL.md @@ -35,7 +35,6 @@ Load these files from `github/gh-aw` (they are not available locally). - `.github/aw/evals.md` - `.github/aw/experiments.md` - `.github/aw/github-agentic-workflows.md` -- `.github/aw/github-mcp-server-pagination.md` - `.github/aw/github-mcp-server.md` - `.github/aw/instructions.md` - `.github/aw/linter-workflows.md` @@ -65,12 +64,10 @@ Load these files from `github/gh-aw` (they are not available locally). - `.github/aw/subagents.md` - `.github/aw/syntax-agentic.md` - `.github/aw/syntax-core.md` -- `.github/aw/syntax-engine.md` - `.github/aw/syntax-tools-imports.md` - `.github/aw/syntax.md` - `.github/aw/test-coverage.md` - `.github/aw/test-expression.md` -- `.github/aw/token-optimization-caching-budgets.md` - `.github/aw/token-optimization.md` - `.github/aw/triggers.md` - `.github/aw/update-agentic-workflow.md`