From 1d8282c62f8e4b98be652c0901e2e74dd71152d4 Mon Sep 17 00:00:00 2001 From: Furkan Date: Tue, 21 Jul 2026 10:28:26 +0300 Subject: [PATCH 1/2] feat: add corpus quality gates to GitHub Action --- action.yml | 54 +++++++++++++++++++++++++++++++++++ tests/action-metadata.test.ts | 15 ++++++++++ 2 files changed, 69 insertions(+) diff --git a/action.yml b/action.yml index 4e2a612..417d2b8 100644 --- a/action.yml +++ b/action.yml @@ -70,6 +70,18 @@ inputs: description: Write bundled validation corpus JSON into the output directory. required: false default: "false" + corpus-metrics-manifest: + description: Optional path to a private corpus metrics manifest to evaluate. + required: false + default: "" + corpus-metrics-baseline: + description: Optional path to a previous corpus metrics JSON report for regression comparison. + required: false + default: "" + corpus-metrics-fail-on-regression: + description: Fail when corpus precision or recall decreases, or false-positive share increases. + required: false + default: "false" contract: description: Write public output contract JSON into the output directory. required: false @@ -118,6 +130,12 @@ outputs: validation-corpus-path: description: Path to the generated validation corpus JSON report when corpus is enabled. value: ${{ steps.run-doctor.outputs.validation-corpus-path }} + corpus-metrics-path: + description: Path to the generated corpus quality metrics JSON report when configured. + value: ${{ steps.run-doctor.outputs.corpus-metrics-path }} + corpus-metrics-diff-path: + description: Path to the generated corpus quality regression JSON report when a baseline is configured. + value: ${{ steps.run-doctor.outputs.corpus-metrics-diff-path }} output-contract-path: description: Path to the generated output contract JSON report when contract is enabled. value: ${{ steps.run-doctor.outputs.output-contract-path }} @@ -141,6 +159,10 @@ runs: - name: Run Codex Plugin Doctor id: run-doctor shell: bash + env: + CORPUS_METRICS_MANIFEST_INPUT: ${{ inputs['corpus-metrics-manifest'] }} + CORPUS_METRICS_BASELINE_INPUT: ${{ inputs['corpus-metrics-baseline'] }} + CORPUS_METRICS_FAIL_ON_REGRESSION_INPUT: ${{ inputs['corpus-metrics-fail-on-regression'] }} run: | set -euo pipefail @@ -149,6 +171,8 @@ runs: json_path="$report_dir/codex-plugin-doctor-report.json" sarif_path="$report_dir/codex-plugin-doctor.sarif" validation_corpus_path="$report_dir/validation-corpus.json" + corpus_metrics_path="$report_dir/corpus-metrics.json" + corpus_metrics_diff_path="$report_dir/corpus-metrics-diff.json" output_contract_path="$report_dir/output-contract.json" action_manifest_path="$report_dir/codex-plugin-doctor-action-manifest.json" review_bundle_path="$report_dir/${{ inputs['review-bundle-dir'] }}" @@ -235,6 +259,28 @@ runs: run_doctor "validation corpus" doctor corpus --json --output "$validation_corpus_path" fi + if [[ -z "$CORPUS_METRICS_MANIFEST_INPUT" ]]; then + if [[ -n "$CORPUS_METRICS_BASELINE_INPUT" || "$CORPUS_METRICS_FAIL_ON_REGRESSION_INPUT" == "true" ]]; then + echo "Corpus metrics baseline and regression gating require corpus-metrics-manifest." + record_status 2 + fi + elif [[ -z "$CORPUS_METRICS_BASELINE_INPUT" && "$CORPUS_METRICS_FAIL_ON_REGRESSION_INPUT" == "true" ]]; then + echo "corpus-metrics-fail-on-regression requires corpus-metrics-baseline." + record_status 2 + else + run_doctor "corpus metrics" doctor corpus metrics --manifest "$CORPUS_METRICS_MANIFEST_INPUT" --json --output "$corpus_metrics_path" + + if [[ -n "$CORPUS_METRICS_BASELINE_INPUT" && -f "$corpus_metrics_path" ]]; then + corpus_metrics_diff_args=(doctor corpus metrics diff --before "$CORPUS_METRICS_BASELINE_INPUT" --after "$corpus_metrics_path" --json --output "$corpus_metrics_diff_path") + + if [[ "$CORPUS_METRICS_FAIL_ON_REGRESSION_INPUT" == "true" ]]; then + corpus_metrics_diff_args+=(--fail-on-regression) + fi + + run_doctor "corpus metrics regression" "${corpus_metrics_diff_args[@]}" + fi + fi + if [[ "${{ inputs.contract }}" == "true" ]]; then run_doctor "output contract" doctor contract --json --output "$output_contract_path" fi @@ -282,6 +328,8 @@ runs: export CODEX_PLUGIN_DOCTOR_ACTION_MARKDOWN="${{ inputs.markdown }}" export CODEX_PLUGIN_DOCTOR_ACTION_SARIF="${{ inputs.sarif }}" export CODEX_PLUGIN_DOCTOR_ACTION_CORPUS="${{ inputs.corpus }}" + export CODEX_PLUGIN_DOCTOR_ACTION_CORPUS_METRICS="$([[ -n "$CORPUS_METRICS_MANIFEST_INPUT" ]] && echo true || echo false)" + export CODEX_PLUGIN_DOCTOR_ACTION_CORPUS_METRICS_DIFF="$([[ -n "$CORPUS_METRICS_BASELINE_INPUT" ]] && echo true || echo false)" export CODEX_PLUGIN_DOCTOR_ACTION_CONTRACT="${{ inputs.contract }}" export CODEX_PLUGIN_DOCTOR_ACTION_REVIEW_BUNDLE="${{ inputs['review-bundle'] }}" export CODEX_PLUGIN_DOCTOR_ACTION_REVIEW_BUNDLE_VERIFY="${{ inputs['review-bundle-verify'] }}" @@ -289,6 +337,8 @@ runs: export CODEX_PLUGIN_DOCTOR_ACTION_JSON_PATH="$json_path" export CODEX_PLUGIN_DOCTOR_ACTION_SARIF_PATH="$sarif_path" export CODEX_PLUGIN_DOCTOR_ACTION_CORPUS_PATH="$validation_corpus_path" + export CODEX_PLUGIN_DOCTOR_ACTION_CORPUS_METRICS_PATH="$corpus_metrics_path" + export CODEX_PLUGIN_DOCTOR_ACTION_CORPUS_METRICS_DIFF_PATH="$corpus_metrics_diff_path" export CODEX_PLUGIN_DOCTOR_ACTION_CONTRACT_PATH="$output_contract_path" export CODEX_PLUGIN_DOCTOR_ACTION_REVIEW_BUNDLE_PATH="$review_bundle_path" export CODEX_PLUGIN_DOCTOR_ACTION_REVIEW_BUNDLE_VERIFICATION_PATH="$review_bundle_verification_path" @@ -318,6 +368,8 @@ runs: json: report("json", "CODEX_PLUGIN_DOCTOR_ACTION_JSON", "CODEX_PLUGIN_DOCTOR_ACTION_JSON_PATH"), sarif: report("sarif", "CODEX_PLUGIN_DOCTOR_ACTION_SARIF", "CODEX_PLUGIN_DOCTOR_ACTION_SARIF_PATH"), corpus: report("corpus", "CODEX_PLUGIN_DOCTOR_ACTION_CORPUS", "CODEX_PLUGIN_DOCTOR_ACTION_CORPUS_PATH"), + corpusMetrics: report("corpusMetrics", "CODEX_PLUGIN_DOCTOR_ACTION_CORPUS_METRICS", "CODEX_PLUGIN_DOCTOR_ACTION_CORPUS_METRICS_PATH"), + corpusMetricsDiff: report("corpusMetricsDiff", "CODEX_PLUGIN_DOCTOR_ACTION_CORPUS_METRICS_DIFF", "CODEX_PLUGIN_DOCTOR_ACTION_CORPUS_METRICS_DIFF_PATH"), contract: report("contract", "CODEX_PLUGIN_DOCTOR_ACTION_CONTRACT", "CODEX_PLUGIN_DOCTOR_ACTION_CONTRACT_PATH"), reviewBundle: report("reviewBundle", "CODEX_PLUGIN_DOCTOR_ACTION_REVIEW_BUNDLE", "CODEX_PLUGIN_DOCTOR_ACTION_REVIEW_BUNDLE_PATH"), reviewBundleVerification: report("reviewBundleVerification", "CODEX_PLUGIN_DOCTOR_ACTION_REVIEW_BUNDLE_VERIFY", "CODEX_PLUGIN_DOCTOR_ACTION_REVIEW_BUNDLE_VERIFICATION_PATH") @@ -340,6 +392,8 @@ runs: echo "json-path=$json_path" echo "sarif-path=$sarif_path" echo "validation-corpus-path=$validation_corpus_path" + echo "corpus-metrics-path=$corpus_metrics_path" + echo "corpus-metrics-diff-path=$corpus_metrics_diff_path" echo "output-contract-path=$output_contract_path" echo "action-manifest-path=$action_manifest_path" echo "review-bundle-path=$review_bundle_path" diff --git a/tests/action-metadata.test.ts b/tests/action-metadata.test.ts index ae199d1..8c2d18f 100644 --- a/tests/action-metadata.test.ts +++ b/tests/action-metadata.test.ts @@ -15,6 +15,8 @@ describe("GitHub Action metadata", () => { expect(actionMetadata).toContain("json-path:"); expect(actionMetadata).toContain("sarif-path:"); expect(actionMetadata).toContain("validation-corpus-path:"); + expect(actionMetadata).toContain("corpus-metrics-path:"); + expect(actionMetadata).toContain("corpus-metrics-diff-path:"); expect(actionMetadata).toContain("output-contract-path:"); expect(actionMetadata).toContain("action-manifest-path:"); expect(actionMetadata).toContain("review-bundle-path:"); @@ -39,6 +41,9 @@ describe("GitHub Action metadata", () => { expect(actionMetadata).toContain("json:"); expect(actionMetadata).toContain("markdown:"); expect(actionMetadata).toContain("corpus:"); + expect(actionMetadata).toContain("corpus-metrics-manifest:"); + expect(actionMetadata).toContain("corpus-metrics-baseline:"); + expect(actionMetadata).toContain("corpus-metrics-fail-on-regression:"); expect(actionMetadata).toContain("contract:"); expect(actionMetadata).toContain("review-bundle:"); expect(actionMetadata).toContain("review-bundle-dir:"); @@ -58,6 +63,9 @@ describe("GitHub Action metadata", () => { expect(actionMetadata).toContain("doctor.github.action.manifest"); expect(actionMetadata).toContain('doctor_version="$(codex-plugin-doctor --version)"'); expect(actionMetadata).toContain('run_doctor "validation corpus" doctor corpus --json --output "$validation_corpus_path"'); + expect(actionMetadata).toContain('run_doctor "corpus metrics" doctor corpus metrics --manifest "$CORPUS_METRICS_MANIFEST_INPUT" --json --output "$corpus_metrics_path"'); + expect(actionMetadata).toContain('corpus_metrics_diff_args=(doctor corpus metrics diff --before "$CORPUS_METRICS_BASELINE_INPUT" --after "$corpus_metrics_path" --json --output "$corpus_metrics_diff_path")'); + expect(actionMetadata).toContain("corpus_metrics_diff_args+=(--fail-on-regression)"); expect(actionMetadata).toContain('run_doctor "output contract" doctor contract --json --output "$output_contract_path"'); expect(actionMetadata).toContain('review_bundle_args=(doctor review-bundle "${{ inputs.path }}" --output "$review_bundle_path" --sign-key-env "$signing_key_env")'); expect(actionMetadata).toContain('doctor review-bundle verify "$review_bundle_path" --target "${{ inputs.path }}" --sign-key-env "$signing_key_env" --json --output "$review_bundle_verification_path"'); @@ -66,6 +74,8 @@ describe("GitHub Action metadata", () => { expect(actionMetadata).toContain('cat "$summary_path" >> "$GITHUB_STEP_SUMMARY"'); expect(actionMetadata).toContain('echo "status=$status"'); expect(actionMetadata).toContain('echo "validation-corpus-path=$validation_corpus_path"'); + expect(actionMetadata).toContain('echo "corpus-metrics-path=$corpus_metrics_path"'); + expect(actionMetadata).toContain('echo "corpus-metrics-diff-path=$corpus_metrics_diff_path"'); expect(actionMetadata).toContain('echo "output-contract-path=$output_contract_path"'); expect(actionMetadata).toContain('echo "action-manifest-path=$action_manifest_path"'); expect(actionMetadata).toContain('echo "review-bundle-path=$review_bundle_path"'); @@ -101,6 +111,11 @@ describe("GitHub Action metadata", () => { expect(actionUsage).toContain("corpus:"); expect(actionUsage).toContain("contract:"); expect(actionUsage).toContain("validation-corpus.json"); + expect(actionUsage).toContain("corpus-metrics.json"); + expect(actionUsage).toContain("corpus-metrics-diff.json"); + expect(actionUsage).toContain("corpus-metrics-manifest:"); + expect(actionUsage).toContain("corpus-metrics-baseline:"); + expect(actionUsage).toContain('corpus-metrics-fail-on-regression: "true"'); expect(actionUsage).toContain("output-contract.json"); expect(actionUsage).toContain("codex-plugin-doctor-action-manifest.json"); expect(actionUsage).toContain("action-manifest-path"); From 973968527a28664d9304f1735d14a46938bf914e Mon Sep 17 00:00:00 2001 From: Furkan Date: Tue, 21 Jul 2026 10:28:31 +0300 Subject: [PATCH 2/2] docs: explain Action corpus regression gates --- README.md | 2 +- docs/guides/github-action.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8ed22a0..5fa5546 100644 --- a/README.md +++ b/README.md @@ -441,7 +441,7 @@ jobs: review-bundle-verify: "true" ``` -The action writes `codex-plugin-doctor-summary.md`, `codex-plugin-doctor-report.json`, `codex-plugin-doctor-action-manifest.json`, optional `codex-plugin-doctor.sarif`, optional `validation-corpus.json`, optional `output-contract.json`, and optional signed `review-bundle/` files to `codex-plugin-doctor-reports`, appends the Markdown report to the GitHub Actions step summary, uploads the report directory as an artifact, and then returns the real validation exit code. Review bundle generation requires a signing key environment variable such as `CODEX_PLUGIN_DOCTOR_SIGNING_KEY`. For runtime probing, SARIF output, corpus and contract artifacts, review bundle artifacts, installed plugin cache checks, CI policy presets, and pinned release examples, see [GitHub Action Usage](./docs/guides/github-action.md). +The action writes `codex-plugin-doctor-summary.md`, `codex-plugin-doctor-report.json`, `codex-plugin-doctor-action-manifest.json`, optional `codex-plugin-doctor.sarif`, optional validation corpus and quality metrics reports, optional `output-contract.json`, and optional signed `review-bundle/` files to `codex-plugin-doctor-reports`, appends the Markdown report to the GitHub Actions step summary, uploads the report directory as an artifact, and then returns the real validation exit code. Review bundle generation requires a signing key environment variable such as `CODEX_PLUGIN_DOCTOR_SIGNING_KEY`. For runtime probing, SARIF output, corpus quality regression gates, corpus and contract artifacts, review bundle artifacts, installed plugin cache checks, CI policy presets, and pinned release examples, see [GitHub Action Usage](./docs/guides/github-action.md). To self-test this repository after cloning it: diff --git a/docs/guides/github-action.md b/docs/guides/github-action.md index 20d5f5c..ef55808 100644 --- a/docs/guides/github-action.md +++ b/docs/guides/github-action.md @@ -90,6 +90,8 @@ The action also exposes these workflow outputs for follow-up steps: - `json-path` - `sarif-path` - `validation-corpus-path` +- `corpus-metrics-path` +- `corpus-metrics-diff-path` - `output-contract-path` - `action-manifest-path` - `review-bundle-path` @@ -130,6 +132,32 @@ The CLI can produce badge output for release notes, README automation, or a stat `--badge-json` follows the Shields endpoint schema with `schemaVersion`, `label`, `message`, and `color`. `--badge-markdown` emits a static shields.io Markdown image link. +## Corpus Quality Gates + +Use a private corpus metrics manifest to measure reviewed precision, recall, and false-positive share in CI. The action writes only the public-safe metrics report into its artifact directory; snapshots, manifest contents, local paths, and review notes are not copied. + +```yaml +- uses: Esquetta/CodexPluginDoctor@v1.49.0 + with: + version: "1.49.0" + path: . + corpus-metrics-manifest: ../private-corpus/metrics.json +``` + +This writes `corpus-metrics.json`. To compare the result with a retained report and fail the job on regression: + +```yaml +- uses: Esquetta/CodexPluginDoctor@v1.49.0 + with: + version: "1.49.0" + path: . + corpus-metrics-manifest: ../private-corpus/metrics.json + corpus-metrics-baseline: .doctor-baselines/corpus-metrics.json + corpus-metrics-fail-on-regression: "true" +``` + +The comparison writes `corpus-metrics-diff.json`. Reports must have the same `corpusDigest`; changed corpus composition is rejected as non-comparable rather than reported as a validator regression. `corpus-metrics-fail-on-regression` requires both a manifest and baseline. + ## History Artifacts Use history output when a workflow should preserve validation trend data between runs.