From d997ccc601c929f9279009bdf0ad45f46ce39dd6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:15:02 +0000 Subject: [PATCH 1/3] plan: ensure experiment JSONL entries include workflow run id Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/architecture-guardian.lock.yml | 1 + .github/workflows/blog-auditor.lock.yml | 1 + .github/workflows/ci-coach.lock.yml | 1 + .github/workflows/daily-doc-healer.lock.yml | 1 + .github/workflows/daily-issues-report.lock.yml | 1 + .github/workflows/daily-news.lock.yml | 1 + .github/workflows/draft-pr-cleanup.lock.yml | 1 + .github/workflows/plan.lock.yml | 1 + .github/workflows/stale-pr-cleanup.lock.yml | 1 + .github/workflows/weekly-blog-post-writer.lock.yml | 1 + 10 files changed, 10 insertions(+) diff --git a/.github/workflows/architecture-guardian.lock.yml b/.github/workflows/architecture-guardian.lock.yml index 2e1612ed77a..14bf25b1f9a 100644 --- a/.github/workflows/architecture-guardian.lock.yml +++ b/.github/workflows/architecture-guardian.lock.yml @@ -1792,6 +1792,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"violations-analyzed","question":"Did the workflow analyze changed Go or JavaScript files from the last 24 hours for architecture violations, or correctly conclude there were none?"},{"id":"issue-created-or-noop","question":"Was an architecture report issue created when violations were found, or was noop or skip behavior used appropriately when there was nothing actionable?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/blog-auditor.lock.yml b/.github/workflows/blog-auditor.lock.yml index 535260cc1d3..73044875ffd 100644 --- a/.github/workflows/blog-auditor.lock.yml +++ b/.github/workflows/blog-auditor.lock.yml @@ -1857,6 +1857,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"audit-completed","question":"Did the workflow complete the blog availability audit, including page access and content validation checks?"},{"id":"discussion-created","question":"Was an audit discussion created summarizing the pass or fail results and suggested remediation when needed?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/ci-coach.lock.yml b/.github/workflows/ci-coach.lock.yml index 6bf5ab0efcf..d3ab17ed7bd 100644 --- a/.github/workflows/ci-coach.lock.yml +++ b/.github/workflows/ci-coach.lock.yml @@ -1843,6 +1843,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"repair-or-optimization-path","question":"Did the workflow check validation-status first and then follow the correct repair or optimization path for this run?"},{"id":"pr-created-or-noop","question":"Was a focused CI improvement pull request created when actionable work was found, or was noop called when CI was already healthy?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-doc-healer.lock.yml b/.github/workflows/daily-doc-healer.lock.yml index b108fc219dd..c7ea55cde8c 100644 --- a/.github/workflows/daily-doc-healer.lock.yml +++ b/.github/workflows/daily-doc-healer.lock.yml @@ -1970,6 +1970,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"gaps-confirmed","question":"Did the workflow identify at least one confirmed documentation gap to fix, or correctly conclude that no actionable gap remained?"},{"id":"pr-issue-or-noop","question":"Was a documentation pull request or issue created for confirmed gaps, or was noop used appropriately when nothing required action?"}]' GH_AW_EVALS_MODEL: "${{ needs.activation.outputs.model_size }}" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-issues-report.lock.yml b/.github/workflows/daily-issues-report.lock.yml index a01527c95c0..4218bec6d89 100644 --- a/.github/workflows/daily-issues-report.lock.yml +++ b/.github/workflows/daily-issues-report.lock.yml @@ -2036,6 +2036,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"report-generated","question":"Did the workflow analyze the prefetched issues data and generate a clustered daily issues report with metrics and trends?"},{"id":"discussion-created","question":"Was a daily issues discussion created successfully with the report findings and recommendations?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-news.lock.yml b/.github/workflows/daily-news.lock.yml index 64da77c50b2..8783ab81823 100644 --- a/.github/workflows/daily-news.lock.yml +++ b/.github/workflows/daily-news.lock.yml @@ -1965,6 +1965,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"digest-generated","question":"Did the workflow analyze the prefetched repository activity and generate a coherent daily news digest?"},{"id":"discussion-created","question":"Was a daily-news discussion created or updated successfully with the report output?"}]' GH_AW_EVALS_MODEL: "copilot/gpt-5.4" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/draft-pr-cleanup.lock.yml b/.github/workflows/draft-pr-cleanup.lock.yml index bc855a84b50..d0a9dc20b9f 100644 --- a/.github/workflows/draft-pr-cleanup.lock.yml +++ b/.github/workflows/draft-pr-cleanup.lock.yml @@ -1765,6 +1765,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"stale-drafts-triaged","question":"Did the workflow review open draft pull requests and classify stale drafts according to the warning and cleanup policy?"},{"id":"warnings-or-closures-applied","question":"Were the expected labels, comments, and closures applied to stale draft pull requests when appropriate?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/plan.lock.yml b/.github/workflows/plan.lock.yml index 51073d47135..f4f285d33b0 100644 --- a/.github/workflows/plan.lock.yml +++ b/.github/workflows/plan.lock.yml @@ -1904,6 +1904,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"subissues-planned","question":"Did the workflow analyze the triggering issue or discussion and break the work into actionable sub-issues?"},{"id":"issue-actions-completed","question":"Were the planned sub-issues created successfully, with discussion closure handled correctly when required?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/stale-pr-cleanup.lock.yml b/.github/workflows/stale-pr-cleanup.lock.yml index b13c8de2d8d..0c71624650e 100644 --- a/.github/workflows/stale-pr-cleanup.lock.yml +++ b/.github/workflows/stale-pr-cleanup.lock.yml @@ -1754,6 +1754,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"stale-prs-triaged","question":"Did the workflow review pull requests open for 30 or more days and classify them according to the cleanup policy?"},{"id":"actions-taken-or-noop","question":"Were the appropriate stale PR comments, labels, and closures applied when needed, or was noop used correctly when no action was required?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/weekly-blog-post-writer.lock.yml b/.github/workflows/weekly-blog-post-writer.lock.yml index e4616091ae7..77ec897fe2b 100644 --- a/.github/workflows/weekly-blog-post-writer.lock.yml +++ b/.github/workflows/weekly-blog-post-writer.lock.yml @@ -1974,6 +1974,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"draft-prepared","question":"Did the workflow analyze recent releases and merged pull requests and prepare a weekly blog post draft?"},{"id":"pr-created-or-explained","question":"Was a blog post pull request created when a draft was ready, or did the agent clearly explain why no pull request was needed?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); From 4c729426759011578adad1346492b40f1d197ca9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:27:33 +0000 Subject: [PATCH 2/3] feat: add experiments.jsonl with action workflow run_id to experiment JSONL entries Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../agent-performance-analyzer.lock.yml | 1 + .../workflows/agent-persona-explorer.lock.yml | 1 + .../workflows/architecture-guardian.lock.yml | 1 + .github/workflows/audit-workflows.lock.yml | 1 + .../aw-failure-investigator.lock.yml | 1 + .github/workflows/blog-auditor.lock.yml | 1 + .../breaking-change-checker.lock.yml | 1 + .github/workflows/ci-coach.lock.yml | 1 + .../workflows/copilot-agent-analysis.lock.yml | 1 + .../daily-agentrx-trace-optimizer.lock.yml | 1 + .../daily-architecture-diagram.lock.yml | 1 + ...strostylelite-markdown-spellcheck.lock.yml | 1 + .../daily-cache-strategy-analyzer.lock.yml | 1 + .../daily-caveman-optimizer.lock.yml | 1 + .github/workflows/daily-code-metrics.lock.yml | 1 + .../daily-community-attribution.lock.yml | 1 + .../workflows/daily-compiler-quality.lock.yml | 1 + .github/workflows/daily-doc-healer.lock.yml | 1 + .github/workflows/daily-doc-updater.lock.yml | 1 + .github/workflows/daily-fact.lock.yml | 1 + .../workflows/daily-issues-report.lock.yml | 1 + .github/workflows/daily-news.lock.yml | 1 + .../daily-safe-output-optimizer.lock.yml | 1 + .../daily-security-red-team.lock.yml | 1 + .github/workflows/daily-semgrep-scan.lock.yml | 1 + .../dataflow-pr-discussion-dataset.lock.yml | 1 + .github/workflows/deep-report.lock.yml | 1 + .../workflows/dependabot-go-checker.lock.yml | 1 + .github/workflows/gpclean.lock.yml | 1 + .github/workflows/issue-arborist.lock.yml | 1 + .github/workflows/plan.lock.yml | 1 + .github/workflows/smoke-antigravity.lock.yml | 1 + .../smoke-copilot-aoai-apikey.lock.yml | 1 + .../smoke-copilot-aoai-entra.lock.yml | 1 + .github/workflows/smoke-copilot.lock.yml | 1 + .github/workflows/smoke-gemini.lock.yml | 1 + .github/workflows/smoke-project.lock.yml | 1 + .github/workflows/smoke-temporary-id.lock.yml | 1 + .../workflows/test-quality-sentinel.lock.yml | 1 + .github/workflows/typist.lock.yml | 1 + .../weekly-blog-post-writer.lock.yml | 1 + .../js/load_experiment_state_from_repo.cjs | 12 +++ actions/setup/js/pick_experiment.cjs | 27 +++++- actions/setup/js/pick_experiment.test.cjs | 89 ++++++++++++++++++- .../experimental/experiments-specification.md | 12 ++- pkg/constants/job_constants.go | 5 ++ pkg/workflow/compiler_experiments.go | 1 + 47 files changed, 184 insertions(+), 3 deletions(-) diff --git a/.github/workflows/agent-performance-analyzer.lock.yml b/.github/workflows/agent-performance-analyzer.lock.yml index dda1591df61..d9e6ccd846c 100644 --- a/.github/workflows/agent-performance-analyzer.lock.yml +++ b/.github/workflows/agent-performance-analyzer.lock.yml @@ -1802,6 +1802,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/agentperformanceanalyzer + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/agent-persona-explorer.lock.yml b/.github/workflows/agent-persona-explorer.lock.yml index cfa1aa18745..8cecdd6f6e8 100644 --- a/.github/workflows/agent-persona-explorer.lock.yml +++ b/.github/workflows/agent-persona-explorer.lock.yml @@ -1710,6 +1710,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/agentpersonaexplorer + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/architecture-guardian.lock.yml b/.github/workflows/architecture-guardian.lock.yml index 14bf25b1f9a..0550dade62a 100644 --- a/.github/workflows/architecture-guardian.lock.yml +++ b/.github/workflows/architecture-guardian.lock.yml @@ -2023,6 +2023,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/architectureguardian + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/audit-workflows.lock.yml b/.github/workflows/audit-workflows.lock.yml index d7752ead297..5deb1661521 100644 --- a/.github/workflows/audit-workflows.lock.yml +++ b/.github/workflows/audit-workflows.lock.yml @@ -1907,6 +1907,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/auditworkflows + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/aw-failure-investigator.lock.yml b/.github/workflows/aw-failure-investigator.lock.yml index 2c6ed265e0e..2dd2c110f98 100644 --- a/.github/workflows/aw-failure-investigator.lock.yml +++ b/.github/workflows/aw-failure-investigator.lock.yml @@ -1852,6 +1852,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/awfailureinvestigator + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/blog-auditor.lock.yml b/.github/workflows/blog-auditor.lock.yml index 73044875ffd..8e04d3dfbca 100644 --- a/.github/workflows/blog-auditor.lock.yml +++ b/.github/workflows/blog-auditor.lock.yml @@ -2029,6 +2029,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/blogauditor + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/breaking-change-checker.lock.yml b/.github/workflows/breaking-change-checker.lock.yml index 932924895d6..92d6ebccf9c 100644 --- a/.github/workflows/breaking-change-checker.lock.yml +++ b/.github/workflows/breaking-change-checker.lock.yml @@ -2020,6 +2020,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/breakingchangechecker + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/ci-coach.lock.yml b/.github/workflows/ci-coach.lock.yml index d3ab17ed7bd..4c177e598ba 100644 --- a/.github/workflows/ci-coach.lock.yml +++ b/.github/workflows/ci-coach.lock.yml @@ -2015,6 +2015,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/cicoach + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/copilot-agent-analysis.lock.yml b/.github/workflows/copilot-agent-analysis.lock.yml index 2b8440d647b..d8e1a856911 100644 --- a/.github/workflows/copilot-agent-analysis.lock.yml +++ b/.github/workflows/copilot-agent-analysis.lock.yml @@ -1778,6 +1778,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/copilotagentanalysis + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-agentrx-trace-optimizer.lock.yml b/.github/workflows/daily-agentrx-trace-optimizer.lock.yml index a96c5b750c1..4739eec31f1 100644 --- a/.github/workflows/daily-agentrx-trace-optimizer.lock.yml +++ b/.github/workflows/daily-agentrx-trace-optimizer.lock.yml @@ -1796,6 +1796,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailyagentrxtraceoptimizer + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-architecture-diagram.lock.yml b/.github/workflows/daily-architecture-diagram.lock.yml index 2aa54fe4320..21f255a1cc6 100644 --- a/.github/workflows/daily-architecture-diagram.lock.yml +++ b/.github/workflows/daily-architecture-diagram.lock.yml @@ -1743,6 +1743,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailyarchitecturediagram + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml b/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml index 196b35e0380..50c0468cc30 100644 --- a/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml +++ b/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml @@ -1690,6 +1690,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailyastrostylelitemarkdownspellcheck + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-cache-strategy-analyzer.lock.yml b/.github/workflows/daily-cache-strategy-analyzer.lock.yml index 011b3ed3198..1e970e22e1d 100644 --- a/.github/workflows/daily-cache-strategy-analyzer.lock.yml +++ b/.github/workflows/daily-cache-strategy-analyzer.lock.yml @@ -1852,6 +1852,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailycachestrategyanalyzer + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-caveman-optimizer.lock.yml b/.github/workflows/daily-caveman-optimizer.lock.yml index f5979c50fc6..8c9689a690f 100644 --- a/.github/workflows/daily-caveman-optimizer.lock.yml +++ b/.github/workflows/daily-caveman-optimizer.lock.yml @@ -1737,6 +1737,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailycavemanoptimizer + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-code-metrics.lock.yml b/.github/workflows/daily-code-metrics.lock.yml index 4c3d992f34f..ec5cb13b57b 100644 --- a/.github/workflows/daily-code-metrics.lock.yml +++ b/.github/workflows/daily-code-metrics.lock.yml @@ -1827,6 +1827,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailycodemetrics + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-community-attribution.lock.yml b/.github/workflows/daily-community-attribution.lock.yml index 5159c0b2884..e9bb1a11ac1 100644 --- a/.github/workflows/daily-community-attribution.lock.yml +++ b/.github/workflows/daily-community-attribution.lock.yml @@ -1764,6 +1764,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailycommunityattribution + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-compiler-quality.lock.yml b/.github/workflows/daily-compiler-quality.lock.yml index f6a13feb462..c149e42dd41 100644 --- a/.github/workflows/daily-compiler-quality.lock.yml +++ b/.github/workflows/daily-compiler-quality.lock.yml @@ -1740,6 +1740,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailycompilerquality + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-doc-healer.lock.yml b/.github/workflows/daily-doc-healer.lock.yml index c7ea55cde8c..a4e41659175 100644 --- a/.github/workflows/daily-doc-healer.lock.yml +++ b/.github/workflows/daily-doc-healer.lock.yml @@ -2142,6 +2142,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailydochealer + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-doc-updater.lock.yml b/.github/workflows/daily-doc-updater.lock.yml index 4f178cfc9c2..bc33660e9a4 100644 --- a/.github/workflows/daily-doc-updater.lock.yml +++ b/.github/workflows/daily-doc-updater.lock.yml @@ -1653,6 +1653,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailydocupdater + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-fact.lock.yml b/.github/workflows/daily-fact.lock.yml index 88a7c918235..c3680a22ab8 100644 --- a/.github/workflows/daily-fact.lock.yml +++ b/.github/workflows/daily-fact.lock.yml @@ -1864,6 +1864,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailyfact + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-issues-report.lock.yml b/.github/workflows/daily-issues-report.lock.yml index 4218bec6d89..7e7fecf2596 100644 --- a/.github/workflows/daily-issues-report.lock.yml +++ b/.github/workflows/daily-issues-report.lock.yml @@ -2208,6 +2208,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailyissuesreport + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-news.lock.yml b/.github/workflows/daily-news.lock.yml index 8783ab81823..916db7be071 100644 --- a/.github/workflows/daily-news.lock.yml +++ b/.github/workflows/daily-news.lock.yml @@ -2137,6 +2137,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailynews + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-safe-output-optimizer.lock.yml b/.github/workflows/daily-safe-output-optimizer.lock.yml index 130470620e9..27e8d9dfd3b 100644 --- a/.github/workflows/daily-safe-output-optimizer.lock.yml +++ b/.github/workflows/daily-safe-output-optimizer.lock.yml @@ -1971,6 +1971,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailysafeoutputoptimizer + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-security-red-team.lock.yml b/.github/workflows/daily-security-red-team.lock.yml index 12751a627b9..0b98067fed8 100644 --- a/.github/workflows/daily-security-red-team.lock.yml +++ b/.github/workflows/daily-security-red-team.lock.yml @@ -1804,6 +1804,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailysecurityredteam + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/daily-semgrep-scan.lock.yml b/.github/workflows/daily-semgrep-scan.lock.yml index bf6e209650b..fa66806c2d1 100644 --- a/.github/workflows/daily-semgrep-scan.lock.yml +++ b/.github/workflows/daily-semgrep-scan.lock.yml @@ -1676,6 +1676,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dailysemgrepscan + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml index 123e8f0b74c..434ad606fae 100644 --- a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml +++ b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml @@ -2007,6 +2007,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dataflowprdiscussiondataset + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/deep-report.lock.yml b/.github/workflows/deep-report.lock.yml index 4da531218cb..618d3aaff17 100644 --- a/.github/workflows/deep-report.lock.yml +++ b/.github/workflows/deep-report.lock.yml @@ -2180,6 +2180,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/deepreport + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/dependabot-go-checker.lock.yml b/.github/workflows/dependabot-go-checker.lock.yml index 0fb87fe079e..ad588b628b1 100644 --- a/.github/workflows/dependabot-go-checker.lock.yml +++ b/.github/workflows/dependabot-go-checker.lock.yml @@ -1665,6 +1665,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/dependabotgochecker + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/gpclean.lock.yml b/.github/workflows/gpclean.lock.yml index 99fe6ec6ee9..6bd7775efd3 100644 --- a/.github/workflows/gpclean.lock.yml +++ b/.github/workflows/gpclean.lock.yml @@ -1701,6 +1701,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/gpclean + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/issue-arborist.lock.yml b/.github/workflows/issue-arborist.lock.yml index 9c94a7d71d6..35fca307fcc 100644 --- a/.github/workflows/issue-arborist.lock.yml +++ b/.github/workflows/issue-arborist.lock.yml @@ -1848,6 +1848,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/issuearborist + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/plan.lock.yml b/.github/workflows/plan.lock.yml index f4f285d33b0..05b87940b84 100644 --- a/.github/workflows/plan.lock.yml +++ b/.github/workflows/plan.lock.yml @@ -2133,6 +2133,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/plan + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/smoke-antigravity.lock.yml b/.github/workflows/smoke-antigravity.lock.yml index 3bf315533c1..5e83e69c28a 100644 --- a/.github/workflows/smoke-antigravity.lock.yml +++ b/.github/workflows/smoke-antigravity.lock.yml @@ -1844,6 +1844,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/smokeantigravity + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/smoke-copilot-aoai-apikey.lock.yml b/.github/workflows/smoke-copilot-aoai-apikey.lock.yml index bc8593ed4df..0b7280fae9f 100644 --- a/.github/workflows/smoke-copilot-aoai-apikey.lock.yml +++ b/.github/workflows/smoke-copilot-aoai-apikey.lock.yml @@ -2911,6 +2911,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/smokecopilotaoaiapikey + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/smoke-copilot-aoai-entra.lock.yml b/.github/workflows/smoke-copilot-aoai-entra.lock.yml index b355917427d..5bf5b5402e1 100644 --- a/.github/workflows/smoke-copilot-aoai-entra.lock.yml +++ b/.github/workflows/smoke-copilot-aoai-entra.lock.yml @@ -2921,6 +2921,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/smokecopilotaoaientra + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index 325ccb8defa..564f430a0a5 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -3217,6 +3217,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/smokecopilot + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/smoke-gemini.lock.yml b/.github/workflows/smoke-gemini.lock.yml index 3fb5a53fbd5..29ffbbb7910 100644 --- a/.github/workflows/smoke-gemini.lock.yml +++ b/.github/workflows/smoke-gemini.lock.yml @@ -1919,6 +1919,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/smokegemini + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/smoke-project.lock.yml b/.github/workflows/smoke-project.lock.yml index 8752ff2464e..bffbf8e393a 100644 --- a/.github/workflows/smoke-project.lock.yml +++ b/.github/workflows/smoke-project.lock.yml @@ -2050,6 +2050,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/smokeproject + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/smoke-temporary-id.lock.yml b/.github/workflows/smoke-temporary-id.lock.yml index a88043d0624..94b78bc09f8 100644 --- a/.github/workflows/smoke-temporary-id.lock.yml +++ b/.github/workflows/smoke-temporary-id.lock.yml @@ -1899,6 +1899,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/smoketemporaryid + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/test-quality-sentinel.lock.yml b/.github/workflows/test-quality-sentinel.lock.yml index b2a052140e5..3ec7f76f4f3 100644 --- a/.github/workflows/test-quality-sentinel.lock.yml +++ b/.github/workflows/test-quality-sentinel.lock.yml @@ -1795,6 +1795,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/testqualitysentinel + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/typist.lock.yml b/.github/workflows/typist.lock.yml index b1aca189fc3..26bf1f7a365 100644 --- a/.github/workflows/typist.lock.yml +++ b/.github/workflows/typist.lock.yml @@ -1727,6 +1727,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/typist + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/weekly-blog-post-writer.lock.yml b/.github/workflows/weekly-blog-post-writer.lock.yml index 77ec897fe2b..e8f717b2c70 100644 --- a/.github/workflows/weekly-blog-post-writer.lock.yml +++ b/.github/workflows/weekly-blog-post-writer.lock.yml @@ -2146,6 +2146,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments GH_AW_EXPERIMENT_BRANCH: experiments/weeklyblogpostwriter + GH_AW_STATE_FILES: state.json,assignments.json,experiments.jsonl with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/actions/setup/js/load_experiment_state_from_repo.cjs b/actions/setup/js/load_experiment_state_from_repo.cjs index fbb78bbafbe..a66e2855ec6 100644 --- a/actions/setup/js/load_experiment_state_from_repo.cjs +++ b/actions/setup/js/load_experiment_state_from_repo.cjs @@ -162,6 +162,18 @@ async function main() { fs.writeFileSync(stateFile, content, "utf8"); core.info(`Experiment state written to ${stateFile}`); + + // Also fetch experiments.jsonl if present so pick_experiment.cjs can append to it. + let experimentsJsonlContent = null; + try { + experimentsJsonlContent = await fetchFileFromBranch(octokit, owner, repo, branch, "experiments.jsonl"); + } catch (/** @type {any} */ err) { + core.warning(`Failed to fetch experiments.jsonl from branch "${branch}": ${getErrorMessage(err)} – will start fresh`); + } + if (experimentsJsonlContent !== null) { + fs.writeFileSync(path.join(stateDir, "experiments.jsonl"), experimentsJsonlContent, "utf8"); + core.info(`Experiment run records written to ${path.join(stateDir, "experiments.jsonl")}`); + } } module.exports = { main, fetchFileFromBranch, validateInputs }; diff --git a/actions/setup/js/pick_experiment.cjs b/actions/setup/js/pick_experiment.cjs index f0613a0e6f4..12979631315 100644 --- a/actions/setup/js/pick_experiment.cjs +++ b/actions/setup/js/pick_experiment.cjs @@ -436,7 +436,8 @@ async function main() { if (!state.runs) { state.runs = []; } - state.runs.push({ run_id: runId, timestamp, assignments: { ...assignments } }); + const runRecord = { run_id: runId, timestamp, assignments: { ...assignments } }; + state.runs.push(runRecord); // Prune run history to avoid state.json growing without bound over many runs. if (state.runs.length > MAX_RUN_HISTORY) { state.runs = state.runs.slice(-MAX_RUN_HISTORY); @@ -451,10 +452,34 @@ async function main() { // OTLP telemetry can read which variant was selected without recomputing it. // Only written when at least one experiment was successfully assigned. if (Object.keys(assignments).length > 0) { + const runId = process.env.GITHUB_RUN_ID || ""; + const timestamp = new Date().toISOString(); + const assignmentsFile = path.join(stateDir, "assignments.json"); fs.writeFileSync(assignmentsFile, JSON.stringify(assignments, null, 2) + "\n", "utf8"); core.info(`Experiment assignments written to ${assignmentsFile}`); + // Append a structured JSONL record so the action workflow run id is always + // present in durable experiment logs. Each line is a self-contained JSON + // object that can be correlated with GitHub Actions run metadata. + const experimentsJsonlFile = path.join(stateDir, "experiments.jsonl"); + let jsonlEntries = []; + try { + const existing = fs.readFileSync(experimentsJsonlFile, "utf8"); + jsonlEntries = existing + .split("\n") + .filter(line => line.trim()) + .map(line => JSON.parse(line)); + } catch { + // File does not exist or is unreadable – start fresh. + } + jsonlEntries.push({ run_id: runId, timestamp, assignments: { ...assignments } }); + if (jsonlEntries.length > MAX_RUN_HISTORY) { + jsonlEntries = jsonlEntries.slice(-MAX_RUN_HISTORY); + } + fs.writeFileSync(experimentsJsonlFile, jsonlEntries.map(r => JSON.stringify(r)).join("\n") + "\n", "utf8"); + core.info(`Experiment run records written to ${experimentsJsonlFile}`); + // Emit OTEL resource attributes so every span in this run carries the // experiment assignments for filtering in Honeycomb/Grafana. const otelAttrs = Object.entries(assignments) diff --git a/actions/setup/js/pick_experiment.test.cjs b/actions/setup/js/pick_experiment.test.cjs index b6ffad96e6b..3e22d35c75f 100644 --- a/actions/setup/js/pick_experiment.test.cjs +++ b/actions/setup/js/pick_experiment.test.cjs @@ -759,7 +759,94 @@ describe("pick_experiment", () => { }); }); - // ── OTEL resource attributes ────────────────────────────────────────────── + // ── experiments.jsonl ──────────────────────────────────────────────────── + + describe("experiments.jsonl", () => { + it("writes experiments.jsonl with run_id when experiments are assigned", async () => { + const stateFile = path.join(tmpDir, "state.json"); + fs.writeFileSync(stateFile, JSON.stringify({ counts: { feat: { X: 1, Y: 0 } }, runs: [] }), "utf8"); + process.env.GH_AW_EXPERIMENT_SPEC = JSON.stringify({ feat: ["X", "Y"] }); + process.env.GH_AW_EXPERIMENT_STATE_FILE = stateFile; + process.env.GH_AW_EXPERIMENT_STATE_DIR = tmpDir; + process.env.GITHUB_RUN_ID = "99"; + + await main(); + + const jsonlFile = path.join(tmpDir, "experiments.jsonl"); + expect(fs.existsSync(jsonlFile)).toBe(true); + const lines = fs + .readFileSync(jsonlFile, "utf8") + .split("\n") + .filter(l => l.trim()); + expect(lines).toHaveLength(1); + const entry = JSON.parse(lines[0]); + expect(entry.run_id).toBe("99"); + expect(entry.assignments).toEqual({ feat: "Y" }); + expect(typeof entry.timestamp).toBe("string"); + }); + + it("does not write experiments.jsonl when no experiments are assigned", async () => { + process.env.GH_AW_EXPERIMENT_SPEC = "{}"; + process.env.GH_AW_EXPERIMENT_STATE_FILE = path.join(tmpDir, "state.json"); + process.env.GH_AW_EXPERIMENT_STATE_DIR = tmpDir; + + await main(); + + const jsonlFile = path.join(tmpDir, "experiments.jsonl"); + expect(fs.existsSync(jsonlFile)).toBe(false); + }); + + it("appends entries to an existing experiments.jsonl across successive runs", async () => { + const stateFile = path.join(tmpDir, "state.json"); + process.env.GH_AW_EXPERIMENT_SPEC = JSON.stringify({ feat: ["X", "Y"] }); + process.env.GH_AW_EXPERIMENT_STATE_FILE = stateFile; + process.env.GH_AW_EXPERIMENT_STATE_DIR = tmpDir; + process.env.GITHUB_RUN_ID = "10"; + + await main(); + vi.clearAllMocks(); + + process.env.GITHUB_RUN_ID = "20"; + await main(); + + const jsonlFile = path.join(tmpDir, "experiments.jsonl"); + const lines = fs + .readFileSync(jsonlFile, "utf8") + .split("\n") + .filter(l => l.trim()); + expect(lines).toHaveLength(2); + expect(JSON.parse(lines[0]).run_id).toBe("10"); + expect(JSON.parse(lines[1]).run_id).toBe("20"); + }); + + it("prunes experiments.jsonl to last MAX_RUN_HISTORY entries", async () => { + const stateFile = path.join(tmpDir, "state.json"); + // Pre-populate experiments.jsonl with 512 entries. + const existingEntries = Array.from({ length: 512 }, (_, i) => ({ + run_id: String(i), + timestamp: "2026-01-01T00:00:00.000Z", + assignments: { feat: "X" }, + })); + const jsonlFile = path.join(tmpDir, "experiments.jsonl"); + fs.writeFileSync(jsonlFile, existingEntries.map(r => JSON.stringify(r)).join("\n") + "\n", "utf8"); + // Also set up state so the pick is deterministic. + fs.writeFileSync(stateFile, JSON.stringify({ counts: { feat: { X: 512, Y: 0 } }, runs: [] }), "utf8"); + process.env.GH_AW_EXPERIMENT_SPEC = JSON.stringify({ feat: ["X", "Y"] }); + process.env.GH_AW_EXPERIMENT_STATE_FILE = stateFile; + process.env.GH_AW_EXPERIMENT_STATE_DIR = tmpDir; + process.env.GITHUB_RUN_ID = "9999"; + + await main(); + + const lines = fs + .readFileSync(jsonlFile, "utf8") + .split("\n") + .filter(l => l.trim()); + // 512 existing + 1 new = 513, pruned to last 512. + expect(lines).toHaveLength(512); + expect(JSON.parse(lines[lines.length - 1]).run_id).toBe("9999"); + }); + }); describe("OTEL resource attributes", () => { it("exports OTEL_RESOURCE_ATTRIBUTES with experiment assignments", async () => { diff --git a/docs/src/content/docs/experimental/experiments-specification.md b/docs/src/content/docs/experimental/experiments-specification.md index 8958303e806..fdb1d5d0613 100644 --- a/docs/src/content/docs/experimental/experiments-specification.md +++ b/docs/src/content/docs/experimental/experiments-specification.md @@ -177,6 +177,7 @@ Notes, rationale blocks, and appendices are **informative** and carry no conform | **Activation job** | The `activation` GitHub Actions job generated by the compiler that picks variants and exposes them to downstream jobs. | | **Experiment artifact** | A GitHub Actions artifact named `{sanitizedID}-experiment` uploaded by the activation job and containing `state.json` and `assignments.json`. | | **assignments.json** | A file in the experiment artifact containing only the current run's variant assignments as a flat JSON object. | +| **experiments.jsonl** | A growing JSONL file committed to the experiments git branch alongside `state.json`. Each line is a JSON object with `run_id`, `timestamp`, and `assignments` for one workflow run, enabling correlation with GitHub Actions run metadata. | --- @@ -429,6 +430,8 @@ date window), implementations **MUST NOT** append a run record or rewrite `state **R-STORE-REPO-001**: When `storage: repo` is active, the activation job **MUST** load experiment state by fetching `state.json` from the git branch named `experiments/{sanitizedWorkflowID}` via the GitHub REST API (GET /repos/{owner}/{repo}/contents/{path}). +The activation job **MUST** also attempt to fetch `experiments.jsonl` from the same branch and, +if present, write it to the local state directory so that `pick_experiment.cjs` can append to it. **R-STORE-REPO-002**: A 404 response (branch or file does not exist) **MUST** be treated as an empty initial state; the activation job **MUST NOT** fail. @@ -436,11 +439,18 @@ empty initial state; the activation job **MUST NOT** fail. **R-STORE-REPO-003**: After the activation job completes, a dedicated `push_experiments_state` job **MUST** be generated. This job **MUST**: - Download the experiment artifact from the current run. -- Commit the updated `state.json` and `assignments.json` to the experiments git branch. +- Commit the updated `state.json`, `assignments.json`, and `experiments.jsonl` to the experiments git branch. - Declare `permissions: contents: write`. - Be listed as a dependency of the conclusion job to ensure state is persisted before the workflow terminates. +**R-STORE-REPO-006**: When at least one experiment is assigned, `pick_experiment.cjs` **MUST** +append a JSONL record to `experiments.jsonl` containing `run_id` (from `GITHUB_RUN_ID`), +`timestamp`, and `assignments`. The file **MUST** be pruned to at most 512 entries (keeping the +most recent). This file provides a durable, run-correlated log that is separate from +`assignments.json` (which contains only current-run data) and `state.json` (which embeds run +history but is primarily an invocation-counter store). + **R-STORE-REPO-004**: The commit **SHOULD** be made via the GitHub GraphQL `createCommitOnBranch` mutation (producing a verified, signed commit). A plain `git push` **MAY** be used as a fallback when the GraphQL mutation is unavailable. diff --git a/pkg/constants/job_constants.go b/pkg/constants/job_constants.go index bce1c542f1c..9b3c8515727 100644 --- a/pkg/constants/job_constants.go +++ b/pkg/constants/job_constants.go @@ -108,6 +108,11 @@ const EvalsResultFilename = "evals.jsonl" // EvalsBranchPrefix is the branch prefix used for durable eval result storage. const EvalsBranchPrefix = "evals" +// ExperimentsRunsFilename is the filename of the experiment run records JSONL file. +// Each line is a JSON object with run_id, timestamp, and assignments for one workflow run. +// Written by pick_experiment.cjs alongside state.json and assignments.json. +const ExperimentsRunsFilename = "experiments.jsonl" + // LegacyDetectionArtifactName is the old artifact name used before the rename. // Kept for backward compatibility when downloading artifacts from older workflow runs. const LegacyDetectionArtifactName = "threat-detection.log" diff --git a/pkg/workflow/compiler_experiments.go b/pkg/workflow/compiler_experiments.go index 2ee6acea5b0..5f2a80a2075 100644 --- a/pkg/workflow/compiler_experiments.go +++ b/pkg/workflow/compiler_experiments.go @@ -703,6 +703,7 @@ func (c *Compiler) buildPushExperimentsStateJob(data *WorkflowData) (*Job, error pushStep.WriteString(" GITHUB_SERVER_URL: ${{ github.server_url }}\n") fmt.Fprintf(&pushStep, " GH_AW_EXPERIMENT_STATE_DIR: %s\n", experimentsCacheDir) fmt.Fprintf(&pushStep, " GH_AW_EXPERIMENT_BRANCH: %s\n", branchName) + fmt.Fprintf(&pushStep, " GH_AW_STATE_FILES: state.json,assignments.json,%s\n", constants.ExperimentsRunsFilename) pushStep.WriteString(" with:\n") pushStep.WriteString(" script: |\n") pushStep.WriteString(" const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n") From a2263a5d7035aab0a3fcea26e388a509cd21c939 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:32:52 +0000 Subject: [PATCH 3/3] refactor: consolidate runId/timestamp in pick_experiment.cjs; clarify load warning Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../js/load_experiment_state_from_repo.cjs | 2 +- actions/setup/js/pick_experiment.cjs | 28 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/actions/setup/js/load_experiment_state_from_repo.cjs b/actions/setup/js/load_experiment_state_from_repo.cjs index a66e2855ec6..f0d7ff0dc53 100644 --- a/actions/setup/js/load_experiment_state_from_repo.cjs +++ b/actions/setup/js/load_experiment_state_from_repo.cjs @@ -168,7 +168,7 @@ async function main() { try { experimentsJsonlContent = await fetchFileFromBranch(octokit, owner, repo, branch, "experiments.jsonl"); } catch (/** @type {any} */ err) { - core.warning(`Failed to fetch experiments.jsonl from branch "${branch}": ${getErrorMessage(err)} – will start fresh`); + core.warning(`Failed to fetch experiments.jsonl from branch "${branch}": ${getErrorMessage(err)} – will start with empty run history`); } if (experimentsJsonlContent !== null) { fs.writeFileSync(path.join(stateDir, "experiments.jsonl"), experimentsJsonlContent, "utf8"); diff --git a/actions/setup/js/pick_experiment.cjs b/actions/setup/js/pick_experiment.cjs index 12979631315..164a532a1b7 100644 --- a/actions/setup/js/pick_experiment.cjs +++ b/actions/setup/js/pick_experiment.cjs @@ -430,31 +430,27 @@ async function main() { core.info(`Experiment assignments (JSON): ${experimentsJSON}`); if (Object.keys(assignments).length > 0) { - // Append a per-run record to state.runs so each assignment is traceable. + // Capture run metadata once and share it across state.runs, experiments.jsonl, and assignments. const runId = process.env.GITHUB_RUN_ID || ""; const timestamp = new Date().toISOString(); + const runRecord = { run_id: runId, timestamp, assignments: { ...assignments } }; + + // Append a per-run record to state.runs so each assignment is traceable. if (!state.runs) { state.runs = []; } - const runRecord = { run_id: runId, timestamp, assignments: { ...assignments } }; state.runs.push(runRecord); // Prune run history to avoid state.json growing without bound over many runs. if (state.runs.length > MAX_RUN_HISTORY) { state.runs = state.runs.slice(-MAX_RUN_HISTORY); } - } - - // Persist updated counts and run history. - saveState(stateFile, state); - core.info(`Experiment state written to ${stateFile}`); - // Persist current-run assignments to a separate file so downstream jobs and - // OTLP telemetry can read which variant was selected without recomputing it. - // Only written when at least one experiment was successfully assigned. - if (Object.keys(assignments).length > 0) { - const runId = process.env.GITHUB_RUN_ID || ""; - const timestamp = new Date().toISOString(); + // Persist updated counts and run history. + saveState(stateFile, state); + core.info(`Experiment state written to ${stateFile}`); + // Persist current-run assignments to a separate file so downstream jobs and + // OTLP telemetry can read which variant was selected without recomputing it. const assignmentsFile = path.join(stateDir, "assignments.json"); fs.writeFileSync(assignmentsFile, JSON.stringify(assignments, null, 2) + "\n", "utf8"); core.info(`Experiment assignments written to ${assignmentsFile}`); @@ -473,7 +469,7 @@ async function main() { } catch { // File does not exist or is unreadable – start fresh. } - jsonlEntries.push({ run_id: runId, timestamp, assignments: { ...assignments } }); + jsonlEntries.push(runRecord); if (jsonlEntries.length > MAX_RUN_HISTORY) { jsonlEntries = jsonlEntries.slice(-MAX_RUN_HISTORY); } @@ -488,6 +484,10 @@ async function main() { const existingAttrs = process.env.OTEL_RESOURCE_ATTRIBUTES || ""; core.exportVariable("OTEL_RESOURCE_ATTRIBUTES", existingAttrs ? `${existingAttrs},${otelAttrs}` : otelAttrs); core.info(`OTEL resource attributes set: ${otelAttrs}`); + } else { + // No assignments – still persist updated counts. + saveState(stateFile, state); + core.info(`Experiment state written to ${stateFile}`); } // Write step summary.