feat(corpus): improve test reporting and add regression tracking #13
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Corpus Parsing Tests | |
| # pull_request_target runs the workflow from the BASE branch (main), | |
| # so a fork PR cannot modify this workflow to exfiltrate secrets. | |
| # The PR's code is checked out explicitly below. | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request_target: | |
| types: [opened, synchronize] | |
| workflow_dispatch: {} | |
| jobs: | |
| corpus: | |
| name: Corpus Parsing Tests | |
| runs-on: ubuntu-latest | |
| # Require approval via environment for fork PRs (secrets access) | |
| environment: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && 'corpus-tests' || '' }} | |
| permissions: | |
| pull-requests: write | |
| actions: read | |
| steps: | |
| # Checkout the PR head (or push sha) — NOT the base branch | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - uses: actions/checkout@v3 | |
| with: | |
| repository: getsynq/kernel-cll-corpus | |
| ssh-key: ${{ secrets.CORPUS_DEPLOY_KEY }} | |
| ref: main | |
| path: tests/corpus-repo | |
| - run: ln -s "$PWD/tests/corpus-repo/corpus" tests/corpus | |
| - uses: dtolnay/rust-toolchain@stable | |
| - name: Run corpus tests | |
| continue-on-error: true | |
| run: cargo test --test sqlparser_corpus | |
| - name: Create fallback report if tests crashed | |
| if: always() | |
| run: | | |
| if [ ! -f target/corpus-report.json ]; then | |
| echo "Warning: corpus-report.json not found (tests may have crashed)" | |
| echo '{"summary":{"total_passed":0,"total_failed":0,"total_tests":0},"by_dialect":{},"test_results":{}}' > target/corpus-report.json | |
| fi | |
| - name: Upload corpus report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: corpus-report | |
| path: target/corpus-report.json | |
| - name: Download baseline from main | |
| if: github.event_name == 'pull_request_target' | |
| id: baseline | |
| continue-on-error: true | |
| uses: dawidd6/action-download-artifact@v6 | |
| with: | |
| name: corpus-report | |
| path: target/baseline | |
| branch: main | |
| workflow: corpus.yml | |
| search_artifacts: true | |
| - name: Generate comparison report | |
| if: github.event_name == 'pull_request_target' | |
| id: generate_report | |
| run: | | |
| BASELINE_ARG="" | |
| if [ -f target/baseline/corpus-report.json ]; then | |
| BASELINE_ARG="target/baseline/corpus-report.json" | |
| fi | |
| node scripts/compare-corpus-reports.js target/corpus-report.json $BASELINE_ARG > target/report.md | |
| - name: Post PR comment | |
| if: github.event_name == 'pull_request_target' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const body = fs.readFileSync('target/report.md', 'utf8'); | |
| // Find existing comment to update | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const marker = '## Corpus Parsing Report'; | |
| const existing = comments.find(c => c.body.startsWith(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body, | |
| }); | |
| } |