From 0fd1967ca01e343284b01d34cc6f62ac3aa328b2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 23 Feb 2026 20:58:11 -0700 Subject: [PATCH] feat: add query + incremental regression benchmarks and README footprint section Add 4 new regression benchmarks (query depth scaling, diff-impact latency, incremental build tiers, import resolution throughput) with dual-engine support, report updaters, and CI workflow jobs. Add lightweight footprint section to README with live shields.io badges for unpacked size, dependency stars, and weekly downloads. --- .github/workflows/benchmark.yml | 126 +++++++++++++++++ CONTRIBUTING.md | 2 + README.md | 14 ++ scripts/incremental-benchmark.js | 202 +++++++++++++++++++++++++++ scripts/query-benchmark.js | 198 ++++++++++++++++++++++++++ scripts/update-incremental-report.js | 150 ++++++++++++++++++++ scripts/update-query-report.js | 144 +++++++++++++++++++ 7 files changed, 836 insertions(+) create mode 100644 scripts/incremental-benchmark.js create mode 100644 scripts/query-benchmark.js create mode 100644 scripts/update-incremental-report.js create mode 100644 scripts/update-query-report.js diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 94e2f131..b6c07b70 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -146,3 +146,129 @@ jobs: --head "$BRANCH" \ --title "docs: update embedding benchmarks" \ --body "Automated embedding benchmark update from workflow run [#${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." + + query-benchmark: + runs-on: ubuntu-latest + if: >- + github.event_name == 'workflow_dispatch' || + github.event.workflow_run.conclusion == 'success' + permissions: + contents: write + pull-requests: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: main + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-node@v4 + with: + node-version: "22" + + - run: npm install + + - name: Run query benchmark + run: node scripts/query-benchmark.js 2>/dev/null > query-benchmark-result.json + + - name: Update query report + run: node scripts/update-query-report.js query-benchmark-result.json + + - name: Upload query result + uses: actions/upload-artifact@v4 + with: + name: query-benchmark-result + path: query-benchmark-result.json + + - name: Check for changes + id: changes + run: | + if git diff --quiet HEAD -- generated/QUERY-BENCHMARKS.md; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Commit and push via PR + if: steps.changes.outputs.changed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + BRANCH="benchmark/query-$(date +%Y%m%d-%H%M%S)" + git checkout -b "$BRANCH" + git add generated/QUERY-BENCHMARKS.md + git commit -m "docs: update query benchmarks" + git push origin "$BRANCH" + + gh pr create \ + --base main \ + --head "$BRANCH" \ + --title "docs: update query benchmarks" \ + --body "Automated query benchmark update from workflow run [#${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." + + incremental-benchmark: + runs-on: ubuntu-latest + if: >- + github.event_name == 'workflow_dispatch' || + github.event.workflow_run.conclusion == 'success' + permissions: + contents: write + pull-requests: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: main + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-node@v4 + with: + node-version: "22" + + - run: npm install + + - name: Run incremental benchmark + run: node scripts/incremental-benchmark.js 2>/dev/null > incremental-benchmark-result.json + + - name: Update incremental report + run: node scripts/update-incremental-report.js incremental-benchmark-result.json + + - name: Upload incremental result + uses: actions/upload-artifact@v4 + with: + name: incremental-benchmark-result + path: incremental-benchmark-result.json + + - name: Check for changes + id: changes + run: | + if git diff --quiet HEAD -- generated/INCREMENTAL-BENCHMARKS.md; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Commit and push via PR + if: steps.changes.outputs.changed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + BRANCH="benchmark/incremental-$(date +%Y%m%d-%H%M%S)" + git checkout -b "$BRANCH" + git add generated/INCREMENTAL-BENCHMARKS.md + git commit -m "docs: update incremental benchmarks" + git push origin "$BRANCH" + + gh pr create \ + --base main \ + --head "$BRANCH" \ + --title "docs: update incremental benchmarks" \ + --body "Automated incremental benchmark update from workflow run [#${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 53ec6a57..34626daf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -153,6 +153,8 @@ description. |-----------|-----------------|-------------| | `node scripts/benchmark.js` | Build speed (native vs WASM), query latency | Changes to `builder.js`, `parser.js`, `queries.js`, `resolve.js`, `db.js`, or the native engine | | `node scripts/embedding-benchmark.js` | Search recall (Hit@1/3/5/10) across models | Changes to `embedder.js` or embedding strategies | +| `node scripts/query-benchmark.js` | Query depth scaling, diff-impact latency | Changes to `queries.js`, `resolve.js`, or `db.js` | +| `node scripts/incremental-benchmark.js` | Incremental build, import resolution throughput | Changes to `builder.js`, `resolve.js`, `parser.js`, or `journal.js` | ### How to report results diff --git a/README.md b/README.md index ddee7f34..011af9d3 100644 --- a/README.md +++ b/README.md @@ -384,6 +384,20 @@ Self-measured on every release via CI ([build benchmarks](generated/BUILD-BENCHM Metrics are normalized per file for cross-version comparability. Times above are for a full initial build — incremental rebuilds only re-parse changed files. +### Lightweight Footprint + +npm unpacked size + +Only **3 runtime dependencies** — everything else is optional or a devDependency: + +| Dependency | What it does | | | +|---|---|---|---| +| [better-sqlite3](https://github.com/WiseLibs/better-sqlite3) | Fast, synchronous SQLite driver | ![GitHub stars](https://img.shields.io/github/stars/WiseLibs/better-sqlite3?style=flat-square&label=%E2%AD%90) | ![npm downloads](https://img.shields.io/npm/dw/better-sqlite3?style=flat-square&label=%F0%9F%93%A5%2Fwk) | +| [commander](https://github.com/tj/commander.js) | CLI argument parsing | ![GitHub stars](https://img.shields.io/github/stars/tj/commander.js?style=flat-square&label=%E2%AD%90) | ![npm downloads](https://img.shields.io/npm/dw/commander?style=flat-square&label=%F0%9F%93%A5%2Fwk) | +| [web-tree-sitter](https://github.com/tree-sitter/tree-sitter) | WASM tree-sitter bindings | ![GitHub stars](https://img.shields.io/github/stars/tree-sitter/tree-sitter?style=flat-square&label=%E2%AD%90) | ![npm downloads](https://img.shields.io/npm/dw/web-tree-sitter?style=flat-square&label=%F0%9F%93%A5%2Fwk) | + +Optional: `@huggingface/transformers` (semantic search), `@modelcontextprotocol/sdk` (MCP server) — lazy-loaded only when needed. + ## 🤖 AI Agent Integration ### MCP Server diff --git a/scripts/incremental-benchmark.js b/scripts/incremental-benchmark.js new file mode 100644 index 00000000..94f4963a --- /dev/null +++ b/scripts/incremental-benchmark.js @@ -0,0 +1,202 @@ +#!/usr/bin/env node + +/** + * Incremental build benchmark — measures build tiers and import resolution. + * + * Measures full build, no-op rebuild, and single-file rebuild for both + * native and WASM engines. Also benchmarks import resolution throughput: + * native batch vs JS fallback. + * + * Usage: node scripts/incremental-benchmark.js > result.json + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const root = path.resolve(__dirname, '..'); + +const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')); +const dbPath = path.join(root, '.codegraph', 'graph.db'); + +const { buildGraph } = await import(pathToFileURL(path.join(root, 'src', 'builder.js')).href); +const { statsData } = await import(pathToFileURL(path.join(root, 'src', 'queries.js')).href); +const { resolveImportPath, resolveImportsBatch, resolveImportPathJS } = await import( + pathToFileURL(path.join(root, 'src', 'resolve.js')).href +); +const { isNativeAvailable } = await import( + pathToFileURL(path.join(root, 'src', 'native.js')).href +); + +// Redirect console.log to stderr so only JSON goes to stdout +const origLog = console.log; +console.log = (...args) => console.error(...args); + +const RUNS = 3; +const PROBE_FILE = path.join(root, 'src', 'queries.js'); + +function median(arr) { + const sorted = [...arr].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +} + +function round1(n) { + return Math.round(n * 10) / 10; +} + +/** + * Benchmark build tiers for a given engine. + */ +async function benchmarkBuildTiers(engine) { + // Full build (delete DB first) + const fullTimings = []; + for (let i = 0; i < RUNS; i++) { + if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); + const start = performance.now(); + await buildGraph(root, { engine, incremental: false }); + fullTimings.push(performance.now() - start); + } + const fullBuildMs = Math.round(median(fullTimings)); + + // No-op rebuild (nothing changed) + const noopTimings = []; + for (let i = 0; i < RUNS; i++) { + const start = performance.now(); + await buildGraph(root, { engine, incremental: true }); + noopTimings.push(performance.now() - start); + } + const noopRebuildMs = Math.round(median(noopTimings)); + + // 1-file change rebuild + const original = fs.readFileSync(PROBE_FILE, 'utf8'); + let oneFileRebuildMs; + try { + const oneFileTimings = []; + for (let i = 0; i < RUNS; i++) { + fs.writeFileSync(PROBE_FILE, original + `\n// probe-${i}\n`); + const start = performance.now(); + await buildGraph(root, { engine, incremental: true }); + oneFileTimings.push(performance.now() - start); + } + oneFileRebuildMs = Math.round(median(oneFileTimings)); + } finally { + fs.writeFileSync(PROBE_FILE, original); + // One final incremental build to restore DB state + await buildGraph(root, { engine, incremental: true }); + } + + return { fullBuildMs, noopRebuildMs, oneFileRebuildMs }; +} + +/** + * Collect all import pairs by scanning source files for ES import statements. + */ +function collectImportPairs() { + const srcDir = path.join(root, 'src'); + const files = fs.readdirSync(srcDir).filter((f) => f.endsWith('.js')); + const importRe = /(?:^|\n)\s*import\s+.*?\s+from\s+['"]([^'"]+)['"]/g; + + const pairs = []; + for (const file of files) { + const absFile = path.join(srcDir, file); + const content = fs.readFileSync(absFile, 'utf8'); + let match; + while ((match = importRe.exec(content)) !== null) { + pairs.push({ fromFile: absFile, importSource: match[1] }); + } + } + return pairs; +} + +/** + * Benchmark import resolution: native batch vs JS fallback. + */ +function benchmarkResolve(inputs) { + const aliases = null; // codegraph itself has no path aliases + + // Native batch + let nativeBatchMs = null; + let perImportNativeMs = null; + if (isNativeAvailable()) { + const timings = []; + for (let i = 0; i < RUNS; i++) { + const start = performance.now(); + resolveImportsBatch(inputs, root, aliases); + timings.push(performance.now() - start); + } + nativeBatchMs = round1(median(timings)); + perImportNativeMs = inputs.length > 0 ? round1(nativeBatchMs / inputs.length) : 0; + } + + // JS fallback (call the exported JS implementation) + const jsTimings = []; + for (let i = 0; i < RUNS; i++) { + const start = performance.now(); + for (const { fromFile, importSource } of inputs) { + resolveImportPathJS(fromFile, importSource, root, aliases); + } + jsTimings.push(performance.now() - start); + } + const jsFallbackMs = round1(median(jsTimings)); + const perImportJsMs = inputs.length > 0 ? round1(jsFallbackMs / inputs.length) : 0; + + return { + imports: inputs.length, + nativeBatchMs, + jsFallbackMs, + perImportNativeMs, + perImportJsMs, + }; +} + +// ── Run benchmarks ─────────────────────────────────────────────────────── + +console.error('Benchmarking WASM engine...'); +const wasm = await benchmarkBuildTiers('wasm'); +console.error(` full=${wasm.fullBuildMs}ms noop=${wasm.noopRebuildMs}ms 1-file=${wasm.oneFileRebuildMs}ms`); + +// Get file count from the WASM-built graph +const stats = statsData(dbPath); +const files = stats.files.total; + +let native = null; +if (isNativeAvailable()) { + console.error('Benchmarking native engine...'); + native = await benchmarkBuildTiers('native'); + console.error(` full=${native.fullBuildMs}ms noop=${native.noopRebuildMs}ms 1-file=${native.oneFileRebuildMs}ms`); +} else { + console.error('Native engine not available — skipping native build benchmark'); +} + +// Import resolution benchmark (uses existing graph) +console.error('Benchmarking import resolution...'); +const inputs = collectImportPairs(); +console.error(` ${inputs.length} import pairs collected`); +const resolve = benchmarkResolve(inputs); +console.error(` native=${resolve.nativeBatchMs}ms js=${resolve.jsFallbackMs}ms`); + +// Restore console.log for JSON output +console.log = origLog; + +const result = { + version: pkg.version, + date: new Date().toISOString().slice(0, 10), + files, + wasm: { + fullBuildMs: wasm.fullBuildMs, + noopRebuildMs: wasm.noopRebuildMs, + oneFileRebuildMs: wasm.oneFileRebuildMs, + }, + native: native + ? { + fullBuildMs: native.fullBuildMs, + noopRebuildMs: native.noopRebuildMs, + oneFileRebuildMs: native.oneFileRebuildMs, + } + : null, + resolve, +}; + +console.log(JSON.stringify(result, null, 2)); diff --git a/scripts/query-benchmark.js b/scripts/query-benchmark.js new file mode 100644 index 00000000..de1716df --- /dev/null +++ b/scripts/query-benchmark.js @@ -0,0 +1,198 @@ +#!/usr/bin/env node + +/** + * Query benchmark runner — measures query depth scaling and diff-impact latency. + * + * Dynamically selects hub/mid/leaf targets from the graph, then benchmarks + * fnDepsData and fnImpactData at depth 1, 3, 5 plus diffImpactData with a + * synthetic staged change. Runs against both native and WASM engine-built + * graphs to catch structural differences. + * + * Usage: node scripts/query-benchmark.js > result.json + */ + +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import Database from 'better-sqlite3'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const root = path.resolve(__dirname, '..'); + +const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')); +const dbPath = path.join(root, '.codegraph', 'graph.db'); + +const { buildGraph } = await import(pathToFileURL(path.join(root, 'src', 'builder.js')).href); +const { fnDepsData, fnImpactData, diffImpactData, statsData } = await import( + pathToFileURL(path.join(root, 'src', 'queries.js')).href +); +const { isNativeAvailable } = await import( + pathToFileURL(path.join(root, 'src', 'native.js')).href +); + +// Redirect console.log to stderr so only JSON goes to stdout +const origLog = console.log; +console.log = (...args) => console.error(...args); + +const RUNS = 5; +const DEPTHS = [1, 3, 5]; + +function median(arr) { + const sorted = [...arr].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +} + +function round1(n) { + return Math.round(n * 10) / 10; +} + +/** + * Select hub / mid / leaf targets dynamically from the graph. + */ +function selectTargets() { + const db = new Database(dbPath, { readonly: true }); + const rows = db + .prepare( + `SELECT n.name, COUNT(e.id) AS cnt + FROM nodes n + JOIN edges e ON e.source_id = n.id OR e.target_id = n.id + WHERE n.file NOT LIKE '%test%' AND n.file NOT LIKE '%spec%' + GROUP BY n.id + ORDER BY cnt DESC`, + ) + .all(); + db.close(); + + if (rows.length === 0) throw new Error('No nodes with edges found in graph'); + + const hub = rows[0].name; + const mid = rows[Math.floor(rows.length / 2)].name; + const leaf = rows[rows.length - 1].name; + return { hub, mid, leaf }; +} + +/** + * Benchmark a single query function at multiple depths. + */ +function benchDepths(fn, name, depths) { + const result = {}; + for (const depth of depths) { + const timings = []; + for (let i = 0; i < RUNS; i++) { + const start = performance.now(); + fn(name, dbPath, { depth, noTests: true }); + timings.push(performance.now() - start); + } + result[`depth${depth}Ms`] = round1(median(timings)); + } + return result; +} + +/** + * Benchmark diff-impact with a synthetic staged change on the hub file. + */ +function benchDiffImpact(hubName) { + // Find the file that contains the hub symbol + const db = new Database(dbPath, { readonly: true }); + const row = db + .prepare(`SELECT file FROM nodes WHERE name = ? LIMIT 1`) + .get(hubName); + db.close(); + + if (!row) return { latencyMs: 0, affectedFunctions: 0, affectedFiles: 0 }; + + const hubFile = path.join(root, row.file); + const original = fs.readFileSync(hubFile, 'utf8'); + + try { + // Append a probe comment and stage it + fs.writeFileSync(hubFile, original + '\n// benchmark-probe\n'); + execFileSync('git', ['add', hubFile], { cwd: root, stdio: 'pipe' }); + + const timings = []; + let lastResult = null; + for (let i = 0; i < RUNS; i++) { + const start = performance.now(); + lastResult = diffImpactData(dbPath, { staged: true, depth: 3, noTests: true }); + timings.push(performance.now() - start); + } + + return { + latencyMs: round1(median(timings)), + affectedFunctions: lastResult?.affectedFunctions?.length || 0, + affectedFiles: lastResult?.affectedFiles?.length || 0, + }; + } finally { + // Restore: unstage + revert content + execFileSync('git', ['restore', '--staged', hubFile], { cwd: root, stdio: 'pipe' }); + fs.writeFileSync(hubFile, original); + } +} + +/** + * Run all query benchmarks against the current graph. + */ +function benchmarkQueries(targets) { + const fnDeps = {}; + const fnImpact = {}; + + // Run depth benchmarks on hub target (most connected — worst case) + fnDeps.depth1Ms = benchDepths(fnDepsData, targets.hub, [1]).depth1Ms; + fnDeps.depth3Ms = benchDepths(fnDepsData, targets.hub, [3]).depth3Ms; + fnDeps.depth5Ms = benchDepths(fnDepsData, targets.hub, [5]).depth5Ms; + + fnImpact.depth1Ms = benchDepths(fnImpactData, targets.hub, [1]).depth1Ms; + fnImpact.depth3Ms = benchDepths(fnImpactData, targets.hub, [3]).depth3Ms; + fnImpact.depth5Ms = benchDepths(fnImpactData, targets.hub, [5]).depth5Ms; + + const diffImpact = benchDiffImpact(targets.hub); + + return { targets, fnDeps, fnImpact, diffImpact }; +} + +// ── Run benchmarks ─────────────────────────────────────────────────────── + +// Build with WASM engine +if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); +await buildGraph(root, { engine: 'wasm', incremental: false }); + +const targets = selectTargets(); +console.error(`Targets: hub=${targets.hub}, mid=${targets.mid}, leaf=${targets.leaf}`); + +const wasm = benchmarkQueries(targets); + +let native = null; +if (isNativeAvailable()) { + if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); + await buildGraph(root, { engine: 'native', incremental: false }); + native = benchmarkQueries(targets); +} else { + console.error('Native engine not available — skipping native benchmark'); +} + +// Restore console.log for JSON output +console.log = origLog; + +const result = { + version: pkg.version, + date: new Date().toISOString().slice(0, 10), + wasm: { + targets: wasm.targets, + fnDeps: wasm.fnDeps, + fnImpact: wasm.fnImpact, + diffImpact: wasm.diffImpact, + }, + native: native + ? { + targets: native.targets, + fnDeps: native.fnDeps, + fnImpact: native.fnImpact, + diffImpact: native.diffImpact, + } + : null, +}; + +console.log(JSON.stringify(result, null, 2)); diff --git a/scripts/update-incremental-report.js b/scripts/update-incremental-report.js new file mode 100644 index 00000000..48d67205 --- /dev/null +++ b/scripts/update-incremental-report.js @@ -0,0 +1,150 @@ +#!/usr/bin/env node + +/** + * Update incremental benchmark report — reads benchmark JSON and updates: + * generated/INCREMENTAL-BENCHMARKS.md (historical table + raw JSON in HTML comment) + * + * Usage: + * node scripts/update-incremental-report.js incremental-benchmark-result.json + * node scripts/incremental-benchmark.js | node scripts/update-incremental-report.js + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const root = path.resolve(__dirname, '..'); + +// ── Read benchmark JSON from file arg or stdin ─────────────────────────── +let jsonText; +const arg = process.argv[2]; +if (arg) { + jsonText = fs.readFileSync(path.resolve(arg), 'utf8'); +} else { + jsonText = fs.readFileSync('/dev/stdin', 'utf8'); +} +const entry = JSON.parse(jsonText); + +// ── Paths ──────────────────────────────────────────────────────────────── +const reportPath = path.join(root, 'generated', 'INCREMENTAL-BENCHMARKS.md'); + +// ── Load existing history ──────────────────────────────────────────────── +let history = []; +if (fs.existsSync(reportPath)) { + const content = fs.readFileSync(reportPath, 'utf8'); + const match = content.match(//); + if (match) { + try { + history = JSON.parse(match[1]); + } catch { + /* start fresh if corrupt */ + } + } +} + +// Add new entry (deduplicate by version — replace if same version exists) +const idx = history.findIndex((h) => h.version === entry.version); +if (idx >= 0) { + history[idx] = entry; +} else { + history.unshift(entry); +} + +// ── Helpers ────────────────────────────────────────────────────────────── +function trend(current, previous, lowerIsBetter = true) { + if (previous == null) return ''; + const pct = ((current - previous) / previous) * 100; + if (Math.abs(pct) < 2) return ' ~'; + if (lowerIsBetter) { + return pct < 0 ? ` ↓${Math.abs(Math.round(pct))}%` : ` ↑${Math.round(pct)}%`; + } + return pct > 0 ? ` ↑${Math.round(pct)}%` : ` ↓${Math.abs(Math.round(pct))}%`; +} + +function formatMs(ms) { + if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`; + return `${Math.round(ms)}ms`; +} + +function engineRow(h, prev, engineKey) { + const e = h[engineKey]; + const p = prev?.[engineKey] || null; + if (!e) return null; + + const fullT = trend(e.fullBuildMs, p?.fullBuildMs); + const noopT = trend(e.noopRebuildMs, p?.noopRebuildMs); + const oneT = trend(e.oneFileRebuildMs, p?.oneFileRebuildMs); + + const r = h.resolve; + const pr = prev?.resolve || null; + const natT = r.nativeBatchMs != null ? trend(r.nativeBatchMs, pr?.nativeBatchMs) : ''; + const jsT = trend(r.jsFallbackMs, pr?.jsFallbackMs); + + return ( + `| ${h.version} | ${engineKey} | ${h.files} ` + + `| ${formatMs(e.fullBuildMs)}${fullT} ` + + `| ${formatMs(e.noopRebuildMs)}${noopT} ` + + `| ${formatMs(e.oneFileRebuildMs)}${oneT} ` + + `| ${r.nativeBatchMs != null ? formatMs(r.nativeBatchMs) + natT : 'n/a'} ` + + `| ${formatMs(r.jsFallbackMs)}${jsT} |` + ); +} + +// ── Build INCREMENTAL-BENCHMARKS.md ────────────────────────────────────── +let md = '# Codegraph Incremental Build Benchmarks\n\n'; +md += 'Self-measured on every release by running codegraph on its own codebase.\n'; +md += 'Build tiers: full (cold), no-op (nothing changed), 1-file (single file modified).\n'; +md += 'Import resolution: native batch vs JS fallback throughput.\n\n'; + +md += + '| Version | Engine | Files | Full Build | No-op | 1-File | Resolve (native) | Resolve (JS) |\n'; +md += + '|---------|--------|------:|-----------:|------:|-------:|------------------:|-------------:|\n'; + +for (let i = 0; i < history.length; i++) { + const h = history[i]; + const prev = history[i + 1] || null; + + const nativeRow = engineRow(h, prev, 'native'); + const wasmRow = engineRow(h, prev, 'wasm'); + if (nativeRow) md += nativeRow + '\n'; + if (wasmRow) md += wasmRow + '\n'; +} + +// ── Latest summary ─────────────────────────────────────────────────────── +const latest = history[0]; +md += '\n### Latest results\n\n'; +md += `**Version:** ${latest.version} | **Files:** ${latest.files} | **Date:** ${latest.date}\n\n`; + +for (const engineKey of ['native', 'wasm']) { + const e = latest[engineKey]; + if (!e) continue; + + md += `#### ${engineKey === 'native' ? 'Native (Rust)' : 'WASM'}\n\n`; + md += '| Metric | Value |\n'; + md += '|--------|------:|\n'; + md += `| Full build | ${formatMs(e.fullBuildMs)} |\n`; + md += `| No-op rebuild | ${formatMs(e.noopRebuildMs)} |\n`; + md += `| 1-file rebuild | ${formatMs(e.oneFileRebuildMs)} |\n\n`; +} + +const r = latest.resolve; +md += '#### Import Resolution\n\n'; +md += '| Metric | Value |\n'; +md += '|--------|------:|\n'; +md += `| Import pairs | ${r.imports} |\n`; +md += `| Native batch | ${r.nativeBatchMs != null ? formatMs(r.nativeBatchMs) : 'n/a'} |\n`; +md += `| JS fallback | ${formatMs(r.jsFallbackMs)} |\n`; +md += `| Per-import (native) | ${r.perImportNativeMs != null ? `${r.perImportNativeMs}ms` : 'n/a'} |\n`; +md += `| Per-import (JS) | ${r.perImportJsMs}ms |\n`; +if (r.nativeBatchMs != null && r.jsFallbackMs > 0) { + md += `| Speedup ratio | ${(r.jsFallbackMs / r.nativeBatchMs).toFixed(1)}x |\n`; +} +md += '\n'; + +md += `\n`; + +fs.mkdirSync(path.dirname(reportPath), { recursive: true }); +fs.writeFileSync(reportPath, md); +console.error(`Updated ${path.relative(root, reportPath)}`); diff --git a/scripts/update-query-report.js b/scripts/update-query-report.js new file mode 100644 index 00000000..08fb1d2e --- /dev/null +++ b/scripts/update-query-report.js @@ -0,0 +1,144 @@ +#!/usr/bin/env node + +/** + * Update query benchmark report — reads benchmark JSON and updates: + * generated/QUERY-BENCHMARKS.md (historical table + raw JSON in HTML comment) + * + * Usage: + * node scripts/update-query-report.js query-benchmark-result.json + * node scripts/query-benchmark.js | node scripts/update-query-report.js + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const root = path.resolve(__dirname, '..'); + +// ── Read benchmark JSON from file arg or stdin ─────────────────────────── +let jsonText; +const arg = process.argv[2]; +if (arg) { + jsonText = fs.readFileSync(path.resolve(arg), 'utf8'); +} else { + jsonText = fs.readFileSync('/dev/stdin', 'utf8'); +} +const entry = JSON.parse(jsonText); + +// ── Paths ──────────────────────────────────────────────────────────────── +const reportPath = path.join(root, 'generated', 'QUERY-BENCHMARKS.md'); + +// ── Load existing history ──────────────────────────────────────────────── +let history = []; +if (fs.existsSync(reportPath)) { + const content = fs.readFileSync(reportPath, 'utf8'); + const match = content.match(//); + if (match) { + try { + history = JSON.parse(match[1]); + } catch { + /* start fresh if corrupt */ + } + } +} + +// Add new entry (deduplicate by version — replace if same version exists) +const idx = history.findIndex((h) => h.version === entry.version); +if (idx >= 0) { + history[idx] = entry; +} else { + history.unshift(entry); +} + +// ── Helpers ────────────────────────────────────────────────────────────── +function trend(current, previous, lowerIsBetter = true) { + if (previous == null) return ''; + const pct = ((current - previous) / previous) * 100; + if (Math.abs(pct) < 2) return ' ~'; + if (lowerIsBetter) { + return pct < 0 ? ` ↓${Math.abs(Math.round(pct))}%` : ` ↑${Math.round(pct)}%`; + } + return pct > 0 ? ` ↑${Math.round(pct)}%` : ` ↓${Math.abs(Math.round(pct))}%`; +} + +function formatMs(ms) { + if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`; + return `${Math.round(ms * 10) / 10}ms`; +} + +function engineRow(h, prev, engineKey) { + const e = h[engineKey]; + const p = prev?.[engineKey] || null; + if (!e) return null; + + const d1t = trend(e.fnDeps.depth1Ms, p?.fnDeps?.depth1Ms); + const d3t = trend(e.fnDeps.depth3Ms, p?.fnDeps?.depth3Ms); + const d5t = trend(e.fnDeps.depth5Ms, p?.fnDeps?.depth5Ms); + const i1t = trend(e.fnImpact.depth1Ms, p?.fnImpact?.depth1Ms); + const i3t = trend(e.fnImpact.depth3Ms, p?.fnImpact?.depth3Ms); + const i5t = trend(e.fnImpact.depth5Ms, p?.fnImpact?.depth5Ms); + const dit = trend(e.diffImpact.latencyMs, p?.diffImpact?.latencyMs); + + return ( + `| ${h.version} | ${engineKey} ` + + `| ${e.fnDeps.depth1Ms}${d1t} ` + + `| ${e.fnDeps.depth3Ms}${d3t} ` + + `| ${e.fnDeps.depth5Ms}${d5t} ` + + `| ${e.fnImpact.depth1Ms}${i1t} ` + + `| ${e.fnImpact.depth3Ms}${i3t} ` + + `| ${e.fnImpact.depth5Ms}${i5t} ` + + `| ${formatMs(e.diffImpact.latencyMs)}${dit} |` + ); +} + +// ── Build QUERY-BENCHMARKS.md ──────────────────────────────────────────── +let md = '# Codegraph Query Benchmarks\n\n'; +md += 'Self-measured on every release by running codegraph queries on its own graph.\n'; +md += 'Latencies are median over 5 runs. Hub target = most-connected node.\n\n'; + +md += + '| Version | Engine | fnDeps d1 | fnDeps d3 | fnDeps d5 | fnImpact d1 | fnImpact d3 | fnImpact d5 | diffImpact |\n'; +md += + '|---------|--------|----------:|----------:|----------:|------------:|------------:|------------:|-----------:|\n'; + +for (let i = 0; i < history.length; i++) { + const h = history[i]; + const prev = history[i + 1] || null; + + const nativeRow = engineRow(h, prev, 'native'); + const wasmRow = engineRow(h, prev, 'wasm'); + if (nativeRow) md += nativeRow + '\n'; + if (wasmRow) md += wasmRow + '\n'; +} + +// ── Latest summary ─────────────────────────────────────────────────────── +const latest = history[0]; +md += '\n### Latest results\n\n'; +md += `**Version:** ${latest.version} | **Date:** ${latest.date}\n\n`; + +for (const engineKey of ['native', 'wasm']) { + const e = latest[engineKey]; + if (!e) continue; + + md += `#### ${engineKey === 'native' ? 'Native (Rust)' : 'WASM'}\n\n`; + md += `**Targets:** hub=\`${e.targets.hub}\`, mid=\`${e.targets.mid}\`, leaf=\`${e.targets.leaf}\`\n\n`; + + md += '| Metric | Value |\n'; + md += '|--------|------:|\n'; + md += `| fnDeps depth 1 | ${formatMs(e.fnDeps.depth1Ms)} |\n`; + md += `| fnDeps depth 3 | ${formatMs(e.fnDeps.depth3Ms)} |\n`; + md += `| fnDeps depth 5 | ${formatMs(e.fnDeps.depth5Ms)} |\n`; + md += `| fnImpact depth 1 | ${formatMs(e.fnImpact.depth1Ms)} |\n`; + md += `| fnImpact depth 3 | ${formatMs(e.fnImpact.depth3Ms)} |\n`; + md += `| fnImpact depth 5 | ${formatMs(e.fnImpact.depth5Ms)} |\n`; + md += `| diffImpact latency | ${formatMs(e.diffImpact.latencyMs)} |\n`; + md += `| diffImpact affected functions | ${e.diffImpact.affectedFunctions} |\n`; + md += `| diffImpact affected files | ${e.diffImpact.affectedFiles} |\n\n`; +} + +md += `\n`; + +fs.mkdirSync(path.dirname(reportPath), { recursive: true }); +fs.writeFileSync(reportPath, md); +console.error(`Updated ${path.relative(root, reportPath)}`);