diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 359e494f9..e0bae12ac 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -17,11 +17,13 @@ jobs: coverage: name: Test Coverage Report runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 15 steps: - name: Checkout repository uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + fetch-depth: 0 - name: Setup Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -35,21 +37,65 @@ jobs: - name: Build project run: npm run build - - name: Run tests with coverage + - name: Run tests with coverage (PR branch) run: npm run test:coverage - - name: Generate coverage summary + - name: Save PR coverage + run: cp coverage/coverage-summary.json /tmp/pr-coverage-summary.json + + - name: Get base branch coverage (PR only) + if: github.event_name == 'pull_request' + id: base_coverage + run: | + # Save the current commit + PR_COMMIT=$(git rev-parse HEAD) + + # Checkout base branch + git checkout ${{ github.event.pull_request.base.sha }} + + # Install dependencies and build for base branch + npm ci + npm run build + + # Run coverage on base branch + npm run test:coverage || true + + # Save base coverage + if [ -f coverage/coverage-summary.json ]; then + cp coverage/coverage-summary.json /tmp/base-coverage-summary.json + echo "base_coverage_exists=true" >> $GITHUB_OUTPUT + else + echo "base_coverage_exists=false" >> $GITHUB_OUTPUT + fi + + # Checkout back to PR commit + git checkout $PR_COMMIT + + # Reinstall PR dependencies + npm ci + + - name: Compare coverage (PR only) + if: github.event_name == 'pull_request' && steps.base_coverage.outputs.base_coverage_exists == 'true' + id: compare + run: | + npx tsx scripts/ci/compare-coverage.ts \ + /tmp/pr-coverage-summary.json \ + /tmp/base-coverage-summary.json + continue-on-error: true + + - name: Generate coverage summary (push to main) + if: github.event_name == 'push' id: coverage run: | # Read the coverage summary COVERAGE_JSON=$(cat coverage/coverage-summary.json) - + # Extract metrics using jq LINES_PCT=$(echo "$COVERAGE_JSON" | jq -r '.total.lines.pct') STATEMENTS_PCT=$(echo "$COVERAGE_JSON" | jq -r '.total.statements.pct') FUNCTIONS_PCT=$(echo "$COVERAGE_JSON" | jq -r '.total.functions.pct') BRANCHES_PCT=$(echo "$COVERAGE_JSON" | jq -r '.total.branches.pct') - + LINES_COVERED=$(echo "$COVERAGE_JSON" | jq -r '.total.lines.covered') LINES_TOTAL=$(echo "$COVERAGE_JSON" | jq -r '.total.lines.total') STATEMENTS_COVERED=$(echo "$COVERAGE_JSON" | jq -r '.total.statements.covered') @@ -58,7 +104,7 @@ jobs: FUNCTIONS_TOTAL=$(echo "$COVERAGE_JSON" | jq -r '.total.functions.total') BRANCHES_COVERED=$(echo "$COVERAGE_JSON" | jq -r '.total.branches.covered') BRANCHES_TOTAL=$(echo "$COVERAGE_JSON" | jq -r '.total.branches.total') - + # Create summary for GitHub Actions Summary echo "## Test Coverage Report" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY @@ -69,62 +115,56 @@ jobs: echo "| **Functions** | ${FUNCTIONS_PCT}% | ${FUNCTIONS_COVERED}/${FUNCTIONS_TOTAL} |" >> $GITHUB_STEP_SUMMARY echo "| **Branches** | ${BRANCHES_PCT}% | ${BRANCHES_COVERED}/${BRANCHES_TOTAL} |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - - # Create PR comment body - COMMENT_BODY="## Test Coverage Report - - | Metric | Coverage | Covered/Total | - |--------|----------|---------------| - | **Lines** | ${LINES_PCT}% | ${LINES_COVERED}/${LINES_TOTAL} | - | **Statements** | ${STATEMENTS_PCT}% | ${STATEMENTS_COVERED}/${STATEMENTS_TOTAL} | - | **Functions** | ${FUNCTIONS_PCT}% | ${FUNCTIONS_COVERED}/${FUNCTIONS_TOTAL} | - | **Branches** | ${BRANCHES_PCT}% | ${BRANCHES_COVERED}/${BRANCHES_TOTAL} | - -
- Coverage Thresholds - - The project has the following coverage thresholds configured: - - Lines: 38% - - Statements: 38% - - Functions: 35% - - Branches: 30% - -
- - --- - *Coverage report generated by \\\`npm run test:coverage\\\`*" - - # Save for next step (escape newlines for GitHub Actions) - echo "COMMENT_BODY<> $GITHUB_ENV - echo "$COMMENT_BODY" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - + # Also save individual metrics as outputs echo "lines_pct=${LINES_PCT}" >> $GITHUB_OUTPUT echo "statements_pct=${STATEMENTS_PCT}" >> $GITHUB_OUTPUT echo "functions_pct=${FUNCTIONS_PCT}" >> $GITHUB_OUTPUT echo "branches_pct=${BRANCHES_PCT}" >> $GITHUB_OUTPUT - - name: Comment PR with coverage report + - name: Comment PR with coverage comparison if: github.event_name == 'pull_request' uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const commentBody = process.env.COMMENT_BODY; - + const fs = require('fs'); + + // Try to read the coverage report from compare step + let commentBody = process.env.COVERAGE_REPORT; + + // If no comparison report, generate a simple report + if (!commentBody) { + const prCoverage = JSON.parse(fs.readFileSync('/tmp/pr-coverage-summary.json', 'utf8')); + const total = prCoverage.total; + + commentBody = `## šŸ“Š Test Coverage Report + + | Metric | Coverage | + |--------|----------| + | Lines | ${total.lines.pct.toFixed(2)}% | + | Statements | ${total.statements.pct.toFixed(2)}% | + | Functions | ${total.functions.pct.toFixed(2)}% | + | Branches | ${total.branches.pct.toFixed(2)}% | + + > ā„¹ļø Base branch coverage not available for comparison. + + --- + *Coverage report generated by \`npm run test:coverage\`*`; + } + // Find existing coverage comment const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, }); - - const botComment = comments.find(comment => - comment.user.type === 'Bot' && - comment.body.includes('Test Coverage Report') + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + (comment.body.includes('Test Coverage Report') || comment.body.includes('Coverage Check')) ); - + if (botComment) { // Update existing comment await github.rest.issues.updateComment({ @@ -151,8 +191,11 @@ jobs: coverage/ retention-days: 30 - - name: Check coverage thresholds + - name: Fail on coverage regression + if: github.event_name == 'pull_request' && steps.compare.outcome == 'failure' run: | - echo "Checking if coverage meets minimum thresholds..." - # Jest will fail if coverage is below thresholds defined in jest.config.js - # This step is informational since the test:coverage command already checks + echo "āŒ Coverage regression detected!" + echo "This PR decreases overall test coverage. Please add tests to maintain coverage levels." + echo "" + echo "See the PR comment above for detailed coverage comparison." + exit 1 diff --git a/scripts/ci/compare-coverage.ts b/scripts/ci/compare-coverage.ts new file mode 100644 index 000000000..171610828 --- /dev/null +++ b/scripts/ci/compare-coverage.ts @@ -0,0 +1,309 @@ +#!/usr/bin/env npx tsx +/** + * Compare coverage between PR and base branch + * + * This script compares coverage-summary.json files from the PR and base branch + * to detect coverage regressions and generate a detailed report. + * + * Usage: + * npx tsx scripts/ci/compare-coverage.ts + * + * Exit codes: + * 0 - Coverage maintained or improved + * 1 - Coverage decreased (regression detected) + */ + +import * as fs from 'fs'; + +interface CoverageMetric { + total: number; + covered: number; + skipped: number; + pct: number; +} + +interface FileCoverage { + lines: CoverageMetric; + statements: CoverageMetric; + functions: CoverageMetric; + branches: CoverageMetric; +} + +interface CoverageSummary { + total: FileCoverage; + [filePath: string]: FileCoverage; +} + +interface CoverageChange { + file: string; + linesBefore: number; + linesAfter: number; + linesDelta: number; + statementsBefore: number; + statementsAfter: number; + statementsDelta: number; +} + +interface ComparisonResult { + overallRegression: boolean; + linesDelta: number; + statementsDelta: number; + functionsDelta: number; + branchesDelta: number; + fileChanges: CoverageChange[]; + newFiles: string[]; + removedFiles: string[]; +} + +function loadCoverageSummary(filePath: string): CoverageSummary | null { + if (!fs.existsSync(filePath)) { + console.error(`Coverage file not found: ${filePath}`); + return null; + } + + try { + const content = fs.readFileSync(filePath, 'utf-8'); + return JSON.parse(content) as CoverageSummary; + } catch (error) { + console.error(`Failed to parse coverage file: ${filePath}`, error); + return null; + } +} + +function compareCoverage( + prCoverage: CoverageSummary, + baseCoverage: CoverageSummary +): ComparisonResult { + const prTotal = prCoverage.total; + const baseTotal = baseCoverage.total; + + // Calculate overall deltas + const linesDelta = prTotal.lines.pct - baseTotal.lines.pct; + const statementsDelta = prTotal.statements.pct - baseTotal.statements.pct; + const functionsDelta = prTotal.functions.pct - baseTotal.functions.pct; + const branchesDelta = prTotal.branches.pct - baseTotal.branches.pct; + + // Check for regression (any overall metric decreased) + // Use a small epsilon to avoid floating point comparison issues + const epsilon = 0.01; + const overallRegression = + linesDelta < -epsilon || + statementsDelta < -epsilon || + functionsDelta < -epsilon || + branchesDelta < -epsilon; + + // Get file lists + const prFiles = new Set(Object.keys(prCoverage).filter((f) => f !== 'total')); + const baseFiles = new Set(Object.keys(baseCoverage).filter((f) => f !== 'total')); + + // Find new, removed, and changed files + const newFiles: string[] = []; + const removedFiles: string[] = []; + const fileChanges: CoverageChange[] = []; + + for (const file of prFiles) { + if (!baseFiles.has(file)) { + newFiles.push(file); + } else { + const prFile = prCoverage[file]; + const baseFile = baseCoverage[file]; + + const linesDiff = prFile.lines.pct - baseFile.lines.pct; + const statementsDiff = prFile.statements.pct - baseFile.statements.pct; + + // Only include files with significant changes + if (Math.abs(linesDiff) > epsilon || Math.abs(statementsDiff) > epsilon) { + fileChanges.push({ + file, + linesBefore: baseFile.lines.pct, + linesAfter: prFile.lines.pct, + linesDelta: linesDiff, + statementsBefore: baseFile.statements.pct, + statementsAfter: prFile.statements.pct, + statementsDelta: statementsDiff, + }); + } + } + } + + for (const file of baseFiles) { + if (!prFiles.has(file)) { + removedFiles.push(file); + } + } + + // Sort file changes by delta (largest decreases first) + fileChanges.sort((a, b) => a.linesDelta - b.linesDelta); + + return { + overallRegression, + linesDelta, + statementsDelta, + functionsDelta, + branchesDelta, + fileChanges, + newFiles, + removedFiles, + }; +} + +function formatDelta(delta: number): string { + const sign = delta >= 0 ? '+' : ''; + return `${sign}${delta.toFixed(2)}%`; +} + +function formatDeltaEmoji(delta: number): string { + if (delta > 0.01) return 'šŸ“ˆ'; + if (delta < -0.01) return 'šŸ“‰'; + return 'āž”ļø'; +} + +function generateMarkdownReport( + prCoverage: CoverageSummary, + baseCoverage: CoverageSummary, + result: ComparisonResult +): string { + const prTotal = prCoverage.total; + const baseTotal = baseCoverage.total; + + let report = ''; + + // Status header + if (result.overallRegression) { + report += `## āš ļø Coverage Regression Detected\n\n`; + report += `This PR decreases test coverage. Please add tests to maintain coverage levels.\n\n`; + } else { + report += `## āœ… Coverage Check Passed\n\n`; + } + + // Overall coverage comparison + report += `### Overall Coverage\n\n`; + report += `| Metric | Base | PR | Delta |\n`; + report += `|--------|------|-----|-------|\n`; + report += `| Lines | ${baseTotal.lines.pct.toFixed(2)}% | ${prTotal.lines.pct.toFixed(2)}% | ${formatDeltaEmoji(result.linesDelta)} ${formatDelta(result.linesDelta)} |\n`; + report += `| Statements | ${baseTotal.statements.pct.toFixed(2)}% | ${prTotal.statements.pct.toFixed(2)}% | ${formatDeltaEmoji(result.statementsDelta)} ${formatDelta(result.statementsDelta)} |\n`; + report += `| Functions | ${baseTotal.functions.pct.toFixed(2)}% | ${prTotal.functions.pct.toFixed(2)}% | ${formatDeltaEmoji(result.functionsDelta)} ${formatDelta(result.functionsDelta)} |\n`; + report += `| Branches | ${baseTotal.branches.pct.toFixed(2)}% | ${prTotal.branches.pct.toFixed(2)}% | ${formatDeltaEmoji(result.branchesDelta)} ${formatDelta(result.branchesDelta)} |\n`; + report += `\n`; + + // Per-file changes + if (result.fileChanges.length > 0) { + report += `
\n`; + report += `šŸ“ Per-file Coverage Changes (${result.fileChanges.length} files)\n\n`; + report += `| File | Lines (Before ā†’ After) | Statements (Before ā†’ After) |\n`; + report += `|------|------------------------|-----------------------------|\n`; + + for (const change of result.fileChanges) { + // Simplify file path for display + const displayPath = change.file.replace(/^.*\/src\//, 'src/'); + const linesChange = `${change.linesBefore.toFixed(1)}% ā†’ ${change.linesAfter.toFixed(1)}% (${formatDelta(change.linesDelta)})`; + const stmtsChange = `${change.statementsBefore.toFixed(1)}% ā†’ ${change.statementsAfter.toFixed(1)}% (${formatDelta(change.statementsDelta)})`; + report += `| \`${displayPath}\` | ${linesChange} | ${stmtsChange} |\n`; + } + + report += `\n
\n\n`; + } + + // New files + if (result.newFiles.length > 0) { + report += `
\n`; + report += `āœØ New Files (${result.newFiles.length} files)\n\n`; + + for (const file of result.newFiles) { + const displayPath = file.replace(/^.*\/src\//, 'src/'); + const coverage = prCoverage[file]; + report += `- \`${displayPath}\`: ${coverage.lines.pct.toFixed(1)}% lines\n`; + } + + report += `\n
\n\n`; + } + + // Removed files + if (result.removedFiles.length > 0) { + report += `
\n`; + report += `šŸ—‘ļø Removed Files (${result.removedFiles.length} files)\n\n`; + + for (const file of result.removedFiles) { + const displayPath = file.replace(/^.*\/src\//, 'src/'); + report += `- \`${displayPath}\`\n`; + } + + report += `\n
\n\n`; + } + + report += `---\n`; + report += `*Coverage comparison generated by \`scripts/ci/compare-coverage.ts\`*\n`; + + return report; +} + +function main(): void { + const args = process.argv.slice(2); + + if (args.length < 2) { + console.error('Usage: compare-coverage.ts '); + console.error(''); + console.error('Arguments:'); + console.error(' pr-coverage.json Path to coverage-summary.json from the PR'); + console.error(' base-coverage.json Path to coverage-summary.json from the base branch'); + process.exit(1); + } + + const prCoveragePath = args[0]; + const baseCoveragePath = args[1]; + + // Load coverage files + const prCoverage = loadCoverageSummary(prCoveragePath); + const baseCoverage = loadCoverageSummary(baseCoveragePath); + + if (!prCoverage || !baseCoverage) { + console.error('Failed to load coverage files'); + process.exit(1); + } + + // Compare coverage + const result = compareCoverage(prCoverage, baseCoverage); + + // Generate report + const report = generateMarkdownReport(prCoverage, baseCoverage, result); + + // Output to GITHUB_STEP_SUMMARY if available + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (summaryPath) { + fs.appendFileSync(summaryPath, report); + console.log('Coverage comparison summary written to GITHUB_STEP_SUMMARY'); + } + + // Also output report to stdout for logs + console.log('\n' + report); + + // Output key metrics as GitHub Actions outputs + const outputPath = process.env.GITHUB_OUTPUT; + if (outputPath) { + fs.appendFileSync( + outputPath, + `regression=${result.overallRegression}\n` + + `lines_delta=${result.linesDelta.toFixed(2)}\n` + + `statements_delta=${result.statementsDelta.toFixed(2)}\n` + + `functions_delta=${result.functionsDelta.toFixed(2)}\n` + + `branches_delta=${result.branchesDelta.toFixed(2)}\n` + ); + } + + // Output the report as an environment variable for PR comment + const envPath = process.env.GITHUB_ENV; + if (envPath) { + fs.appendFileSync(envPath, `COVERAGE_REPORT<