From cf6ab57c3429006220a7b64927b66c6d4e23b824 Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Wed, 18 Mar 2026 23:14:06 -0600 Subject: [PATCH 1/7] feat: child-process isolation for benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run each engine/model benchmark in a forked subprocess so that segfaults (e.g. from the native Rust addon) only kill the child — the parent survives and collects partial results from whichever engines succeeded. - New shared utility: scripts/lib/fork-engine.js (isWorker, workerEngine, forkEngines) handles subprocess lifecycle, timeout, crash recovery, and partial JSON result extraction - benchmark.js, query-benchmark.js, incremental-benchmark.js: fork per engine (wasm/native) via forkEngines() - embedding-benchmark.js: fork per model (ONNX runtime can OOM/segfault) - JSON output contract unchanged — report scripts and CI need no changes --- scripts/benchmark.js | 307 +++++++++++++---------------- scripts/embedding-benchmark.js | 194 ++++++++++++------- scripts/incremental-benchmark.js | 319 ++++++++++++++----------------- scripts/lib/fork-engine.js | 163 ++++++++++++++++ scripts/query-benchmark.js | 155 ++++++--------- 5 files changed, 630 insertions(+), 508 deletions(-) create mode 100644 scripts/lib/fork-engine.js diff --git a/scripts/benchmark.js b/scripts/benchmark.js index 7b8c0c05..c2651443 100644 --- a/scripts/benchmark.js +++ b/scripts/benchmark.js @@ -3,8 +3,9 @@ /** * Benchmark runner — measures codegraph performance on itself (dogfooding). * - * Runs both native (Rust) and WASM engines, outputs JSON to stdout - * with raw and per-file normalized metrics for each. + * Each engine (native / WASM) runs in a forked subprocess so that a segfault + * in the native addon only kills the child — the parent survives and collects + * partial results from whichever engines succeeded. * * Usage: node scripts/benchmark.js */ @@ -15,25 +16,73 @@ import { performance } from 'node:perf_hooks'; import { fileURLToPath } from 'node:url'; import Database from 'better-sqlite3'; import { resolveBenchmarkSource, srcImport } from './lib/bench-config.js'; +import { isWorker, workerEngine, forkEngines } from './lib/fork-engine.js'; + +// ── Parent process: fork one child per engine, assemble final output ───── +if (!isWorker()) { + const { version } = await resolveBenchmarkSource(); + const { wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2)); + + const primary = wasm || native; + if (!primary) { + console.error('Error: Both engines failed. No results to report.'); + process.exit(1); + } + + const result = { + version, + date: new Date().toISOString().slice(0, 10), + files: primary.files, + wasm: wasm + ? { + buildTimeMs: wasm.buildTimeMs, + queryTimeMs: wasm.queryTimeMs, + nodes: wasm.nodes, + edges: wasm.edges, + dbSizeBytes: wasm.dbSizeBytes, + perFile: wasm.perFile, + noopRebuildMs: wasm.noopRebuildMs, + oneFileRebuildMs: wasm.oneFileRebuildMs, + oneFilePhases: wasm.oneFilePhases, + queries: wasm.queries, + phases: wasm.phases, + } + : null, + native: native + ? { + buildTimeMs: native.buildTimeMs, + queryTimeMs: native.queryTimeMs, + nodes: native.nodes, + edges: native.edges, + dbSizeBytes: native.dbSizeBytes, + perFile: native.perFile, + noopRebuildMs: native.noopRebuildMs, + oneFileRebuildMs: native.oneFileRebuildMs, + oneFilePhases: native.oneFilePhases, + queries: native.queries, + phases: native.phases, + } + : null, + }; + + console.log(JSON.stringify(result, null, 2)); + process.exit(0); +} + +// ── Worker process: benchmark a single engine, write JSON to stdout ────── +const engine = workerEngine(); const __dirname = path.dirname(fileURLToPath(import.meta.url)); const root = path.resolve(__dirname, '..'); -const { version, srcDir, cleanup } = await resolveBenchmarkSource(); +const { srcDir, cleanup } = await resolveBenchmarkSource(); const dbPath = path.join(root, '.codegraph', 'graph.db'); -// Import programmatic API (use file:// URLs for Windows compatibility) const { buildGraph } = await import(srcImport(srcDir, 'builder.js')); const { fnDepsData, fnImpactData, pathData, rolesData, statsData } = await import( srcImport(srcDir, 'queries.js') ); -const { isNativeAvailable } = await import( - srcImport(srcDir, 'native.js') -); -const { isWasmAvailable } = await import( - srcImport(srcDir, 'parser.js') -); const INCREMENTAL_RUNS = 3; const QUERY_RUNS = 5; @@ -49,9 +98,6 @@ function round1(n) { return Math.round(n * 10) / 10; } -/** - * Pick hub (most-connected) and leaf (least-connected) non-test symbols from the DB. - */ function selectTargets() { const db = new Database(dbPath, { readonly: true }); const rows = db @@ -67,7 +113,6 @@ function selectTargets() { db.close(); if (rows.length === 0) return { hub: 'buildGraph', leaf: 'median' }; - return { hub: rows[0].name, leaf: rows[rows.length - 1].name }; } @@ -75,175 +120,99 @@ function selectTargets() { const origLog = console.log; console.log = (...args) => console.error(...args); -async function benchmarkEngine(engine) { - // Clean DB for a full build - if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); - - const buildStart = performance.now(); - const buildResult = await buildGraph(root, { engine, incremental: false }); - const buildTimeMs = performance.now() - buildStart; - - const queryStart = performance.now(); - fnDepsData('buildGraph', dbPath); - const queryTimeMs = performance.now() - queryStart; - - const stats = statsData(dbPath); - const totalFiles = stats.files.total; - const totalNodes = stats.nodes.total; - const totalEdges = stats.edges.total; - const dbSizeBytes = fs.statSync(dbPath).size; - - // ── Incremental build tiers (reuse existing DB from full build) ───── - console.error(` [${engine}] Benchmarking no-op rebuild...`); - const noopTimings = []; +// Clean DB for a full build +if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); + +const buildStart = performance.now(); +const buildResult = await buildGraph(root, { engine, incremental: false }); +const buildTimeMs = performance.now() - buildStart; + +const queryStart = performance.now(); +fnDepsData('buildGraph', dbPath); +const queryTimeMs = performance.now() - queryStart; + +const stats = statsData(dbPath); +const totalFiles = stats.files.total; +const totalNodes = stats.nodes.total; +const totalEdges = stats.edges.total; +const dbSizeBytes = fs.statSync(dbPath).size; + +// ── Incremental build tiers ───────────────────────────────────────── +console.error(` [${engine}] Benchmarking no-op rebuild...`); +const noopTimings = []; +for (let i = 0; i < INCREMENTAL_RUNS; i++) { + const start = performance.now(); + await buildGraph(root, { engine, incremental: true }); + noopTimings.push(performance.now() - start); +} +const noopRebuildMs = Math.round(median(noopTimings)); + +console.error(` [${engine}] Benchmarking 1-file rebuild...`); +const original = fs.readFileSync(PROBE_FILE, 'utf8'); +let oneFileRebuildMs; +let oneFilePhases = null; +try { + const oneFileRuns = []; for (let i = 0; i < INCREMENTAL_RUNS; i++) { + fs.writeFileSync(PROBE_FILE, original + `\n// probe-${i}\n`); const start = performance.now(); - await buildGraph(root, { engine, incremental: true }); - noopTimings.push(performance.now() - start); - } - const noopRebuildMs = Math.round(median(noopTimings)); - - console.error(` [${engine}] Benchmarking 1-file rebuild...`); - const original = fs.readFileSync(PROBE_FILE, 'utf8'); - let oneFileRebuildMs; - let oneFilePhases = null; - try { - const oneFileRuns = []; - for (let i = 0; i < INCREMENTAL_RUNS; i++) { - fs.writeFileSync(PROBE_FILE, original + `\n// probe-${i}\n`); - const start = performance.now(); - const res = await buildGraph(root, { engine, incremental: true }); - oneFileRuns.push({ ms: performance.now() - start, phases: res?.phases || null }); - } - oneFileRuns.sort((a, b) => a.ms - b.ms); - const medianRun = oneFileRuns[Math.floor(oneFileRuns.length / 2)]; - oneFileRebuildMs = Math.round(medianRun.ms); - oneFilePhases = medianRun.phases; - } finally { - fs.writeFileSync(PROBE_FILE, original); - await buildGraph(root, { engine, incremental: true }); - } - - // ── Query benchmarks (median of QUERY_RUNS each) ──────────────────── - console.error(` [${engine}] Benchmarking queries...`); - const targets = selectTargets(); - console.error(` hub=${targets.hub}, leaf=${targets.leaf}`); - - function benchQuery(fn, ...args) { - const timings = []; - for (let i = 0; i < QUERY_RUNS; i++) { - const start = performance.now(); - fn(...args); - timings.push(performance.now() - start); - } - return round1(median(timings)); + const res = await buildGraph(root, { engine, incremental: true }); + oneFileRuns.push({ ms: performance.now() - start, phases: res?.phases || null }); } - - const queries = { - fnDepsMs: fnDepsData ? benchQuery(fnDepsData, targets.hub, dbPath, { depth: 3, noTests: true }) : null, - fnImpactMs: fnImpactData ? benchQuery(fnImpactData, targets.hub, dbPath, { depth: 3, noTests: true }) : null, - pathMs: pathData ? benchQuery(pathData, targets.hub, targets.leaf, dbPath, { noTests: true }) : null, - rolesMs: rolesData ? benchQuery(rolesData, dbPath, { noTests: true }) : null, - }; - - return { - buildTimeMs: Math.round(buildTimeMs), - queryTimeMs: Math.round(queryTimeMs * 10) / 10, - nodes: totalNodes, - edges: totalEdges, - files: totalFiles, - dbSizeBytes, - perFile: { - buildTimeMs: Math.round((buildTimeMs / totalFiles) * 10) / 10, - nodes: Math.round((totalNodes / totalFiles) * 10) / 10, - edges: Math.round((totalEdges / totalFiles) * 10) / 10, - dbSizeBytes: Math.round(dbSizeBytes / totalFiles), - }, - noopRebuildMs, - oneFileRebuildMs, - oneFilePhases, - queries, - phases: buildResult?.phases || null, - }; + oneFileRuns.sort((a, b) => a.ms - b.ms); + const medianRun = oneFileRuns[Math.floor(oneFileRuns.length / 2)]; + oneFileRebuildMs = Math.round(medianRun.ms); + oneFilePhases = medianRun.phases; +} finally { + fs.writeFileSync(PROBE_FILE, original); + await buildGraph(root, { engine, incremental: true }); } -// ── Run benchmarks ─────────────────────────────────────────────────────── -const hasWasm = isWasmAvailable(); -const hasNative = isNativeAvailable(); - -if (!hasWasm && !hasNative) { - console.error('Error: Neither WASM grammars nor native engine are available.'); - console.error('Run "npm run build:wasm" to build WASM grammars, or install the native platform package.'); - process.exit(1); -} +// ── Query benchmarks ──────────────────────────────────────────────── +console.error(` [${engine}] Benchmarking queries...`); +const targets = selectTargets(); +console.error(` hub=${targets.hub}, leaf=${targets.leaf}`); -let wasm = null; -if (hasWasm) { - try { - wasm = await benchmarkEngine('wasm'); - } catch (err) { - console.error(`WASM benchmark failed: ${err?.message ?? String(err)}`); +function benchQuery(fn, ...args) { + const timings = []; + for (let i = 0; i < QUERY_RUNS; i++) { + const start = performance.now(); + fn(...args); + timings.push(performance.now() - start); } -} else { - console.error('WASM grammars not built — skipping WASM benchmark'); + return round1(median(timings)); } -let native = null; -if (hasNative) { - try { - native = await benchmarkEngine('native'); - } catch (err) { - console.error(`Native benchmark failed: ${err?.message ?? String(err)}`); - } -} else { - console.error('Native engine not available — skipping native benchmark'); -} +const queries = { + fnDepsMs: fnDepsData ? benchQuery(fnDepsData, targets.hub, dbPath, { depth: 3, noTests: true }) : null, + fnImpactMs: fnImpactData ? benchQuery(fnImpactData, targets.hub, dbPath, { depth: 3, noTests: true }) : null, + pathMs: pathData ? benchQuery(pathData, targets.hub, targets.leaf, dbPath, { noTests: true }) : null, + rolesMs: rolesData ? benchQuery(rolesData, dbPath, { noTests: true }) : null, +}; // Restore console.log for JSON output console.log = origLog; -const primary = wasm || native; -if (!primary) { - console.error('Error: Both engines failed. No results to report.'); - cleanup(); - process.exit(1); -} -const result = { - version, - date: new Date().toISOString().slice(0, 10), - files: primary.files, - wasm: wasm - ? { - buildTimeMs: wasm.buildTimeMs, - queryTimeMs: wasm.queryTimeMs, - nodes: wasm.nodes, - edges: wasm.edges, - dbSizeBytes: wasm.dbSizeBytes, - perFile: wasm.perFile, - noopRebuildMs: wasm.noopRebuildMs, - oneFileRebuildMs: wasm.oneFileRebuildMs, - oneFilePhases: wasm.oneFilePhases, - queries: wasm.queries, - phases: wasm.phases, - } - : null, - native: native - ? { - buildTimeMs: native.buildTimeMs, - queryTimeMs: native.queryTimeMs, - nodes: native.nodes, - edges: native.edges, - dbSizeBytes: native.dbSizeBytes, - perFile: native.perFile, - noopRebuildMs: native.noopRebuildMs, - oneFileRebuildMs: native.oneFileRebuildMs, - oneFilePhases: native.oneFilePhases, - queries: native.queries, - phases: native.phases, - } - : null, +const workerResult = { + buildTimeMs: Math.round(buildTimeMs), + queryTimeMs: Math.round(queryTimeMs * 10) / 10, + nodes: totalNodes, + edges: totalEdges, + files: totalFiles, + dbSizeBytes, + perFile: { + buildTimeMs: Math.round((buildTimeMs / totalFiles) * 10) / 10, + nodes: Math.round((totalNodes / totalFiles) * 10) / 10, + edges: Math.round((totalEdges / totalFiles) * 10) / 10, + dbSizeBytes: Math.round(dbSizeBytes / totalFiles), + }, + noopRebuildMs, + oneFileRebuildMs, + oneFilePhases, + queries, + phases: buildResult?.phases || null, }; -console.log(JSON.stringify(result, null, 2)); +console.log(JSON.stringify(workerResult)); cleanup(); diff --git a/scripts/embedding-benchmark.js b/scripts/embedding-benchmark.js index 4bc3afec..35344011 100644 --- a/scripts/embedding-benchmark.js +++ b/scripts/embedding-benchmark.js @@ -3,70 +3,76 @@ /** * Embedding benchmark runner — measures search recall across all models. * - * For every function/method/class in the graph, generates a query from the - * symbol name (splitIdentifier) and checks if search finds that symbol. - * Tests all available embedding models, outputs JSON to stdout. - * - * Skips jina-code when HF_TOKEN is not set (gated model). + * Each model runs in a forked subprocess so that a crash (OOM, WASM segfault + * in the ONNX runtime) only kills the child — the parent survives and collects + * partial results from whichever models succeeded. * * Usage: node scripts/embedding-benchmark.js > result.json */ -import fs from 'node:fs'; +import { fork } from 'node:child_process'; import path from 'node:path'; import { performance } from 'node:perf_hooks'; import { fileURLToPath } from 'node:url'; import Database from 'better-sqlite3'; import { resolveBenchmarkSource, srcImport } from './lib/bench-config.js'; +const MODEL_WORKER_KEY = '__BENCH_MODEL__'; + const __dirname = path.dirname(fileURLToPath(import.meta.url)); const root = path.resolve(__dirname, '..'); -const { version, srcDir, cleanup } = await resolveBenchmarkSource(); -const dbPath = path.join(root, '.codegraph', 'graph.db'); +// ── Worker process: benchmark a single model, write JSON to stdout ─────── +if (process.env[MODEL_WORKER_KEY]) { + const modelKey = process.env[MODEL_WORKER_KEY]; -const { buildEmbeddings, MODELS, searchData, disposeModel } = await import( - srcImport(srcDir, 'embeddings/index.js') -); + const { srcDir, cleanup } = await resolveBenchmarkSource(); + const dbPath = path.join(root, '.codegraph', 'graph.db'); -// Redirect console.log to stderr so only JSON goes to stdout -const origLog = console.log; -console.log = (...args) => console.error(...args); + const { buildEmbeddings, MODELS, searchData, disposeModel } = await import( + srcImport(srcDir, 'embeddings/index.js') + ); -const TEST_PATTERN = /\.(test|spec)\.|__test__|__tests__|\.stories\./; + const TEST_PATTERN = /\.(test|spec)\.|__test__|__tests__|\.stories\./; -function splitIdentifier(name) { - return name - .replace(/([a-z])([A-Z])/g, '$1 $2') - .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') - .replace(/[_-]+/g, ' ') - .trim(); -} + function splitIdentifier(name) { + return name + .replace(/([a-z])([A-Z])/g, '$1 $2') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') + .replace(/[_-]+/g, ' ') + .trim(); + } -function loadSymbols() { - const db = new Database(dbPath, { readonly: true }); - let rows = db - .prepare( - `SELECT name, kind, file FROM nodes WHERE kind IN ('function', 'method', 'class') ORDER BY file, line`, - ) - .all(); - db.close(); - - rows = rows.filter((r) => !TEST_PATTERN.test(r.file)); - - const seen = new Set(); - const symbols = []; - for (const row of rows) { - if (seen.has(row.name)) continue; - seen.add(row.name); - const query = splitIdentifier(row.name); - if (query.length < 4) continue; - symbols.push({ name: row.name, kind: row.kind, file: row.file, query }); + function loadSymbols() { + const db = new Database(dbPath, { readonly: true }); + let rows = db + .prepare( + `SELECT name, kind, file FROM nodes WHERE kind IN ('function', 'method', 'class') ORDER BY file, line`, + ) + .all(); + db.close(); + + rows = rows.filter((r) => !TEST_PATTERN.test(r.file)); + + const seen = new Set(); + const symbols = []; + for (const row of rows) { + if (seen.has(row.name)) continue; + seen.add(row.name); + const query = splitIdentifier(row.name); + if (query.length < 4) continue; + symbols.push({ name: row.name, kind: row.kind, file: row.file, query }); + } + return symbols; } - return symbols; -} -async function benchmarkModel(modelKey, symbols) { + // Redirect console.log to stderr so only JSON goes to stdout + const origLog = console.log; + console.log = (...args) => console.error(...args); + + const symbols = loadSymbols(); + console.error(` [${modelKey}] Loaded ${symbols.length} symbols`); + const embedStart = performance.now(); await buildEmbeddings(root, modelKey, dbPath, { strategy: 'structured' }); const embedTimeMs = Math.round(performance.now() - embedStart); @@ -90,8 +96,10 @@ async function benchmarkModel(modelKey, symbols) { } const searchTimeMs = Math.round(performance.now() - searchStart); + try { await disposeModel(); } catch { /* best-effort */ } + const total = symbols.length; - return { + const modelResult = { dim: MODELS[modelKey].dim, contextWindow: MODELS[modelKey].contextWindow, hits1, @@ -103,16 +111,82 @@ async function benchmarkModel(modelKey, symbols) { embedTimeMs, searchTimeMs, }; + + console.log = origLog; + console.log(JSON.stringify({ symbols: symbols.length, result: modelResult })); + + cleanup(); + process.exit(0); } -// ── Run benchmarks ────────────────────────────────────────────────────── +// ── Parent process: fork one child per model, assemble final output ────── +const { version, srcDir, cleanup } = await resolveBenchmarkSource(); +const dbPath = path.join(root, '.codegraph', 'graph.db'); -const symbols = loadSymbols(); -console.error(`Loaded ${symbols.length} symbols for benchmark`); +const { MODELS } = await import(srcImport(srcDir, 'embeddings/index.js')); +const TIMEOUT_MS = 600_000; const hasHfToken = !!process.env.HF_TOKEN; const modelKeys = Object.keys(MODELS); const results = {}; +let symbolCount = 0; + +const scriptPath = fileURLToPath(import.meta.url); + +function forkModel(modelKey) { + return new Promise((resolve) => { + console.error(`\n[fork] Spawning ${modelKey} worker (pid isolation)...`); + + const child = fork(scriptPath, process.argv.slice(2), { + env: { ...process.env, [MODEL_WORKER_KEY]: modelKey }, + stdio: ['ignore', 'pipe', 'inherit', 'ipc'], + timeout: TIMEOUT_MS, + }); + + let stdout = ''; + child.stdout.on('data', (chunk) => { stdout += chunk; }); + + const timer = setTimeout(() => { + console.error(`[fork] ${modelKey} worker timed out after ${TIMEOUT_MS / 1000}s — killing`); + child.kill('SIGKILL'); + }, TIMEOUT_MS); + + child.on('close', (code, signal) => { + clearTimeout(timer); + + if (signal) { + console.error(`[fork] ${modelKey} worker killed by signal ${signal}`); + resolve(null); + return; + } + + if (code !== 0) { + console.error(`[fork] ${modelKey} worker exited with code ${code}`); + try { + const parsed = JSON.parse(stdout); + console.error(`[fork] ${modelKey} worker produced partial results despite non-zero exit`); + resolve(parsed); + } catch { + resolve(null); + } + return; + } + + try { + resolve(JSON.parse(stdout)); + } catch (err) { + console.error(`[fork] ${modelKey} worker produced invalid JSON: ${err.message}`); + resolve(null); + } + }); + + child.on('error', (err) => { + clearTimeout(timer); + console.error(`[fork] ${modelKey} worker failed to start: ${err.message}`); + resolve(null); + }); + }); +} for (const key of modelKeys) { if (key === 'jina-code' && !hasHfToken) { @@ -120,32 +194,24 @@ for (const key of modelKeys) { continue; } - console.error(`\nBenchmarking model: ${key}...`); - try { - results[key] = await benchmarkModel(key, symbols); - const r = results[key]; + const data = await forkModel(key); + if (data) { + results[key] = data.result; + if (data.symbols) symbolCount = data.symbols; + const r = data.result; console.error( ` Hit@1=${r.hits1}/${r.total} Hit@3=${r.hits3}/${r.total} Hit@5=${r.hits5}/${r.total} misses=${r.misses}`, ); - } catch (err) { - console.error(` FAILED: ${err?.message ?? String(err)}`); - } finally { - try { - await disposeModel(); - } catch (disposeErr) { - console.error(` disposeModel failed: ${disposeErr?.message ?? String(disposeErr)}`); - } + } else { + console.error(` ${key}: FAILED (worker crashed or timed out)`); } } -// Restore console.log for JSON output -console.log = origLog; - const output = { version, date: new Date().toISOString().slice(0, 10), strategy: 'structured', - symbols: symbols.length, + symbols: symbolCount, models: results, }; diff --git a/scripts/incremental-benchmark.js b/scripts/incremental-benchmark.js index bc20b208..94c3ac9b 100644 --- a/scripts/incremental-benchmark.js +++ b/scripts/incremental-benchmark.js @@ -3,9 +3,9 @@ /** * Incremental build benchmark — measures build tiers and import resolution. * - * Measures full build, no-op rebuild, and single-file rebuild for both - * native and WASM engines. Also benchmarks import resolution throughput: - * native batch vs JS fallback. + * Each engine (native / WASM) runs in a forked subprocess so that a segfault + * in the native addon only kills the child — the parent survives and collects + * partial results from whichever engines succeeded. * * Usage: node scripts/incremental-benchmark.js > result.json */ @@ -15,216 +15,185 @@ import path from 'node:path'; import { performance } from 'node:perf_hooks'; import { fileURLToPath } from 'node:url'; import { resolveBenchmarkSource, srcImport } from './lib/bench-config.js'; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const root = path.resolve(__dirname, '..'); - -const { version, srcDir, cleanup } = await resolveBenchmarkSource(); -const dbPath = path.join(root, '.codegraph', 'graph.db'); - -const { buildGraph } = await import(srcImport(srcDir, 'builder.js')); -const { statsData } = await import(srcImport(srcDir, 'queries.js')); -const { resolveImportPath, resolveImportsBatch, resolveImportPathJS } = await import( - srcImport(srcDir, 'resolve.js') -); -const { isNativeAvailable } = await import( - srcImport(srcDir, 'native.js') -); -const { isWasmAvailable } = await import( - srcImport(srcDir, 'parser.js') -); - -// Redirect console.log to stderr so only JSON goes to stdout -const origLog = console.log; -console.log = (...args) => console.error(...args); - -const RUNS = 3; -const PROBE_FILE = path.join(root, 'src', 'queries.js'); - -function median(arr) { - const sorted = [...arr].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; -} - -function round1(n) { - return Math.round(n * 10) / 10; -} - -/** - * Benchmark build tiers for a given engine. - */ -async function benchmarkBuildTiers(engine) { - // Full build (delete DB first) - const fullTimings = []; - for (let i = 0; i < RUNS; i++) { - if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); - const start = performance.now(); - await buildGraph(root, { engine, incremental: false }); - fullTimings.push(performance.now() - start); - } - const fullBuildMs = Math.round(median(fullTimings)); - - // No-op rebuild (nothing changed) - const noopTimings = []; - for (let i = 0; i < RUNS; i++) { - const start = performance.now(); - await buildGraph(root, { engine, incremental: true }); - noopTimings.push(performance.now() - start); +import { isWorker, workerEngine, forkEngines } from './lib/fork-engine.js'; + +// ── Parent process: fork one child per engine, assemble final output ───── +if (!isWorker()) { + const { version, srcDir: parentSrcDir, cleanup: parentCleanup } = await resolveBenchmarkSource(); + const { wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2)); + + // Import resolution runs in the parent — it tests both native and JS + // fallback in a single pass and doesn't need engine isolation. + const __dirParent = path.dirname(fileURLToPath(import.meta.url)); + const rootParent = path.resolve(__dirParent, '..'); + const dbPathParent = path.join(rootParent, '.codegraph', 'graph.db'); + + const { statsData: parentStats } = await import(srcImport(parentSrcDir, 'queries.js')); + const { resolveImportsBatch: parentBatch, resolveImportPathJS: parentJS } = await import( + srcImport(parentSrcDir, 'resolve.js') + ); + const { isNativeAvailable: parentNativeCheck } = await import( + srcImport(parentSrcDir, 'native.js') + ); + + const RUNS = 3; + function median(arr) { + const sorted = [...arr].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; } - const noopRebuildMs = Math.round(median(noopTimings)); - - // 1-file change rebuild - const original = fs.readFileSync(PROBE_FILE, 'utf8'); - let oneFileRebuildMs; - let oneFilePhases = null; - try { - const oneFileRuns = []; - for (let i = 0; i < RUNS; i++) { - fs.writeFileSync(PROBE_FILE, original + `\n// probe-${i}\n`); - const start = performance.now(); - const res = await buildGraph(root, { engine, incremental: true }); - oneFileRuns.push({ ms: performance.now() - start, phases: res?.phases || null }); + function round1(n) { return Math.round(n * 10) / 10; } + + function collectImportPairs() { + const srcDir = path.join(rootParent, 'src'); + const files = fs.readdirSync(srcDir).filter((f) => f.endsWith('.js')); + const importRe = /(?:^|\n)\s*import\s+.*?\s+from\s+['"]([^'"]+)['"]/g; + const pairs = []; + for (const file of files) { + const absFile = path.join(srcDir, file); + const content = fs.readFileSync(absFile, 'utf8'); + let match; + while ((match = importRe.exec(content)) !== null) { + pairs.push({ fromFile: absFile, importSource: match[1] }); + } } - oneFileRuns.sort((a, b) => a.ms - b.ms); - const medianRun = oneFileRuns[Math.floor(oneFileRuns.length / 2)]; - oneFileRebuildMs = Math.round(medianRun.ms); - oneFilePhases = medianRun.phases; - } finally { - fs.writeFileSync(PROBE_FILE, original); - // One final incremental build to restore DB state - await buildGraph(root, { engine, incremental: true }); + return pairs; } - return { fullBuildMs, noopRebuildMs, oneFileRebuildMs, oneFilePhases }; -} + let stats = null; + try { stats = parentStats(dbPathParent); } catch { /* DB may not exist if both engines failed */ } + const files = stats?.files?.total ?? (wasm?.files || native?.files || 0); -/** - * Collect all import pairs by scanning source files for ES import statements. - */ -function collectImportPairs() { - const srcDir = path.join(root, 'src'); - const files = fs.readdirSync(srcDir).filter((f) => f.endsWith('.js')); - const importRe = /(?:^|\n)\s*import\s+.*?\s+from\s+['"]([^'"]+)['"]/g; - - const pairs = []; - for (const file of files) { - const absFile = path.join(srcDir, file); - const content = fs.readFileSync(absFile, 'utf8'); - let match; - while ((match = importRe.exec(content)) !== null) { - pairs.push({ fromFile: absFile, importSource: match[1] }); - } - } - return pairs; -} + console.error('Benchmarking import resolution...'); + const inputs = collectImportPairs(); + console.error(` ${inputs.length} import pairs collected`); -/** - * Benchmark import resolution: native batch vs JS fallback. - */ -function benchmarkResolve(inputs) { - const aliases = null; // codegraph itself has no path aliases - - // Native batch let nativeBatchMs = null; let perImportNativeMs = null; - if (isNativeAvailable()) { + if (parentNativeCheck()) { const timings = []; for (let i = 0; i < RUNS; i++) { const start = performance.now(); - resolveImportsBatch(inputs, root, aliases); + parentBatch(inputs, rootParent, null); timings.push(performance.now() - start); } nativeBatchMs = round1(median(timings)); perImportNativeMs = inputs.length > 0 ? round1(nativeBatchMs / inputs.length) : 0; } - - // JS fallback (call the exported JS implementation) const jsTimings = []; for (let i = 0; i < RUNS; i++) { const start = performance.now(); for (const { fromFile, importSource } of inputs) { - resolveImportPathJS(fromFile, importSource, root, aliases); + parentJS(fromFile, importSource, rootParent, null); } jsTimings.push(performance.now() - start); } const jsFallbackMs = round1(median(jsTimings)); const perImportJsMs = inputs.length > 0 ? round1(jsFallbackMs / inputs.length) : 0; - return { - imports: inputs.length, - nativeBatchMs, - jsFallbackMs, - perImportNativeMs, - perImportJsMs, + const resolve = { imports: inputs.length, nativeBatchMs, jsFallbackMs, perImportNativeMs, perImportJsMs }; + console.error(` native=${resolve.nativeBatchMs}ms js=${resolve.jsFallbackMs}ms`); + + const result = { + version, + date: new Date().toISOString().slice(0, 10), + files, + wasm: wasm + ? { + fullBuildMs: wasm.fullBuildMs, + noopRebuildMs: wasm.noopRebuildMs, + oneFileRebuildMs: wasm.oneFileRebuildMs, + oneFilePhases: wasm.oneFilePhases, + } + : null, + native: native + ? { + fullBuildMs: native.fullBuildMs, + noopRebuildMs: native.noopRebuildMs, + oneFileRebuildMs: native.oneFileRebuildMs, + oneFilePhases: native.oneFilePhases, + } + : null, + resolve, }; + + console.log(JSON.stringify(result, null, 2)); + parentCleanup(); + process.exit(0); } -// ── Run benchmarks ─────────────────────────────────────────────────────── -const hasWasm = isWasmAvailable(); -const hasNative = isNativeAvailable(); +// ── Worker process: benchmark build tiers for a single engine ──────────── +const engine = workerEngine(); -if (!hasWasm && !hasNative) { - console.error('Error: Neither WASM grammars nor native engine are available.'); - console.error('Run "npm run build:wasm" to build WASM grammars, or install the native platform package.'); - process.exit(1); -} +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const root = path.resolve(__dirname, '..'); -let wasm = null; -if (hasWasm) { - console.error('Benchmarking WASM engine...'); - wasm = await benchmarkBuildTiers('wasm'); - console.error(` full=${wasm.fullBuildMs}ms noop=${wasm.noopRebuildMs}ms 1-file=${wasm.oneFileRebuildMs}ms`); -} else { - console.error('WASM grammars not built — skipping WASM benchmark'); -} +const { srcDir, cleanup } = await resolveBenchmarkSource(); +const dbPath = path.join(root, '.codegraph', 'graph.db'); + +const { buildGraph } = await import(srcImport(srcDir, 'builder.js')); + +// Redirect console.log to stderr so only JSON goes to stdout +const origLog = console.log; +console.log = (...args) => console.error(...args); -let native = null; -if (hasNative) { - console.error('Benchmarking native engine...'); - native = await benchmarkBuildTiers('native'); - console.error(` full=${native.fullBuildMs}ms noop=${native.noopRebuildMs}ms 1-file=${native.oneFileRebuildMs}ms`); -} else { - console.error('Native engine not available — skipping native build benchmark'); +const RUNS = 3; +const PROBE_FILE = path.join(root, 'src', 'queries.js'); + +function median(arr) { + const sorted = [...arr].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; } -// Get file count from whichever graph was built last -const stats = statsData(dbPath); -const files = stats.files.total; +console.error(`Benchmarking ${engine} engine...`); + +// Full build (delete DB first) +const fullTimings = []; +for (let i = 0; i < RUNS; i++) { + if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); + const start = performance.now(); + await buildGraph(root, { engine, incremental: false }); + fullTimings.push(performance.now() - start); +} +const fullBuildMs = Math.round(median(fullTimings)); + +// No-op rebuild (nothing changed) +const noopTimings = []; +for (let i = 0; i < RUNS; i++) { + const start = performance.now(); + await buildGraph(root, { engine, incremental: true }); + noopTimings.push(performance.now() - start); +} +const noopRebuildMs = Math.round(median(noopTimings)); + +// 1-file change rebuild +const original = fs.readFileSync(PROBE_FILE, 'utf8'); +let oneFileRebuildMs; +let oneFilePhases = null; +try { + const oneFileRuns = []; + for (let i = 0; i < RUNS; i++) { + fs.writeFileSync(PROBE_FILE, original + `\n// probe-${i}\n`); + const start = performance.now(); + const res = await buildGraph(root, { engine, incremental: true }); + oneFileRuns.push({ ms: performance.now() - start, phases: res?.phases || null }); + } + oneFileRuns.sort((a, b) => a.ms - b.ms); + const medianRun = oneFileRuns[Math.floor(oneFileRuns.length / 2)]; + oneFileRebuildMs = Math.round(medianRun.ms); + oneFilePhases = medianRun.phases; +} finally { + fs.writeFileSync(PROBE_FILE, original); + await buildGraph(root, { engine, incremental: true }); +} -// Import resolution benchmark (uses existing graph) -console.error('Benchmarking import resolution...'); -const inputs = collectImportPairs(); -console.error(` ${inputs.length} import pairs collected`); -const resolve = benchmarkResolve(inputs); -console.error(` native=${resolve.nativeBatchMs}ms js=${resolve.jsFallbackMs}ms`); +console.error(` full=${fullBuildMs}ms noop=${noopRebuildMs}ms 1-file=${oneFileRebuildMs}ms`); // Restore console.log for JSON output console.log = origLog; -const result = { - version, - date: new Date().toISOString().slice(0, 10), - files, - wasm: wasm - ? { - fullBuildMs: wasm.fullBuildMs, - noopRebuildMs: wasm.noopRebuildMs, - oneFileRebuildMs: wasm.oneFileRebuildMs, - oneFilePhases: wasm.oneFilePhases, - } - : null, - native: native - ? { - fullBuildMs: native.fullBuildMs, - noopRebuildMs: native.noopRebuildMs, - oneFileRebuildMs: native.oneFileRebuildMs, - oneFilePhases: native.oneFilePhases, - } - : null, - resolve, -}; - -console.log(JSON.stringify(result, null, 2)); +const workerResult = { fullBuildMs, noopRebuildMs, oneFileRebuildMs, oneFilePhases }; +console.log(JSON.stringify(workerResult)); cleanup(); diff --git a/scripts/lib/fork-engine.js b/scripts/lib/fork-engine.js new file mode 100644 index 00000000..d0594777 --- /dev/null +++ b/scripts/lib/fork-engine.js @@ -0,0 +1,163 @@ +/** + * Child-process isolation for benchmarks. + * + * Runs each engine benchmark in a subprocess so that segfaults (e.g. from the + * native Rust addon) only kill the child — the parent survives and collects + * partial results from whichever engines succeeded. + * + * Usage (in a benchmark script): + * + * import { forkEngines, isWorker, workerEngine } from './lib/fork-engine.js'; + * + * if (isWorker()) { + * // Child path — run a single engine, write JSON to stdout, then exit. + * const engine = workerEngine(); + * const result = await runBenchmarkForEngine(engine); + * process.stdout.write(JSON.stringify(result)); + * process.exit(0); + * } + * + * // Parent path — fork one child per engine, collect results. + * const { wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2)); + */ + +import { fork } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const WORKER_ENV_KEY = '__BENCH_ENGINE__'; + +/** + * Returns true when running inside a forked worker process. + */ +export function isWorker() { + return !!process.env[WORKER_ENV_KEY]; +} + +/** + * Returns the engine name ('wasm' | 'native') assigned to this worker. + * Throws if called outside a worker. + */ +export function workerEngine() { + const engine = process.env[WORKER_ENV_KEY]; + if (!engine) throw new Error('workerEngine() called outside a worker process'); + return engine; +} + +/** + * Fork the calling script once per available engine, collect JSON results. + * + * @param {string} scriptUrl import.meta.url of the calling benchmark script + * @param {string[]} argv CLI args to forward (e.g. ['--version', '1.0.0', '--npm']) + * @param {object} [opts] + * @param {number} [opts.timeoutMs=600_000] Per-engine timeout (default 10 min) + * @returns {Promise<{ wasm: object|null, native: object|null }>} + */ +export async function forkEngines(scriptUrl, argv = [], opts = {}) { + const scriptPath = fileURLToPath(scriptUrl); + const timeoutMs = opts.timeoutMs ?? 600_000; + + // Detect available engines by importing the check functions in-process. + // These are lightweight checks (no parsing), safe to run in the parent. + let hasWasm = false; + let hasNative = false; + + // We need srcDir to resolve the imports. Re-use bench-config for this. + const { resolveBenchmarkSource, srcImport } = await import('./bench-config.js'); + const { srcDir, cleanup } = await resolveBenchmarkSource(); + + try { + const { isWasmAvailable } = await import(srcImport(srcDir, 'parser.js')); + hasWasm = isWasmAvailable(); + } catch { /* unavailable */ } + + try { + const { isNativeAvailable } = await import(srcImport(srcDir, 'native.js')); + hasNative = isNativeAvailable(); + } catch { /* unavailable */ } + + cleanup(); + + if (!hasWasm && !hasNative) { + console.error('Error: Neither WASM grammars nor native engine are available.'); + console.error('Run "npm run build:wasm" to build WASM grammars, or install the native platform package.'); + process.exit(1); + } + + /** + * Fork a single engine worker and collect its JSON output. + * @param {string} engine + * @returns {Promise} + */ + function runWorker(engine) { + return new Promise((resolve) => { + console.error(`\n[fork] Spawning ${engine} worker (pid isolation)...`); + + const child = fork(scriptPath, argv, { + env: { ...process.env, [WORKER_ENV_KEY]: engine }, + stdio: ['ignore', 'pipe', 'inherit', 'ipc'], + timeout: timeoutMs, + }); + + let stdout = ''; + child.stdout.on('data', (chunk) => { stdout += chunk; }); + + const timer = setTimeout(() => { + console.error(`[fork] ${engine} worker timed out after ${timeoutMs / 1000}s — killing`); + child.kill('SIGKILL'); + }, timeoutMs); + + child.on('close', (code, signal) => { + clearTimeout(timer); + + if (signal) { + console.error(`[fork] ${engine} worker killed by signal ${signal}`); + resolve(null); + return; + } + + if (code !== 0) { + console.error(`[fork] ${engine} worker exited with code ${code}`); + // Try to parse partial output anyway + try { + const parsed = JSON.parse(stdout); + console.error(`[fork] ${engine} worker produced partial results despite non-zero exit`); + resolve(parsed); + } catch { + resolve(null); + } + return; + } + + try { + resolve(JSON.parse(stdout)); + } catch (err) { + console.error(`[fork] ${engine} worker produced invalid JSON: ${err.message}`); + resolve(null); + } + }); + + child.on('error', (err) => { + clearTimeout(timer); + console.error(`[fork] ${engine} worker failed to start: ${err.message}`); + resolve(null); + }); + }); + } + + const results = { wasm: null, native: null }; + + // Run engines sequentially — they share the DB file and filesystem state. + if (hasWasm) { + results.wasm = await runWorker('wasm'); + } else { + console.error('WASM grammars not built — skipping WASM benchmark'); + } + + if (hasNative) { + results.native = await runWorker('native'); + } else { + console.error('Native engine not available — skipping native benchmark'); + } + + return results; +} diff --git a/scripts/query-benchmark.js b/scripts/query-benchmark.js index 76dd9151..0360040c 100644 --- a/scripts/query-benchmark.js +++ b/scripts/query-benchmark.js @@ -3,10 +3,9 @@ /** * Query benchmark runner — measures query depth scaling and diff-impact latency. * - * Dynamically selects hub/mid/leaf targets from the graph, then benchmarks - * fnDepsData and fnImpactData at depth 1, 3, 5 plus diffImpactData with a - * synthetic staged change. Runs against both native and WASM engine-built - * graphs to catch structural differences. + * Each engine (native / WASM) runs in a forked subprocess so that a segfault + * in the native addon only kills the child — the parent survives and collects + * partial results from whichever engines succeeded. * * Usage: node scripts/query-benchmark.js > result.json */ @@ -18,30 +17,57 @@ import { performance } from 'node:perf_hooks'; import { fileURLToPath } from 'node:url'; import Database from 'better-sqlite3'; import { resolveBenchmarkSource, srcImport } from './lib/bench-config.js'; +import { isWorker, workerEngine, forkEngines } from './lib/fork-engine.js'; + +// ── Parent process: fork one child per engine, assemble final output ───── +if (!isWorker()) { + const { version } = await resolveBenchmarkSource(); + const { wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2)); + + const result = { + version, + date: new Date().toISOString().slice(0, 10), + wasm: wasm + ? { + targets: wasm.targets, + fnDeps: wasm.fnDeps, + fnImpact: wasm.fnImpact, + diffImpact: wasm.diffImpact, + } + : null, + native: native + ? { + targets: native.targets, + fnDeps: native.fnDeps, + fnImpact: native.fnImpact, + diffImpact: native.diffImpact, + } + : null, + }; + + console.log(JSON.stringify(result, null, 2)); + process.exit(0); +} + +// ── Worker process: benchmark a single engine, write JSON to stdout ────── +const engine = workerEngine(); const __dirname = path.dirname(fileURLToPath(import.meta.url)); const root = path.resolve(__dirname, '..'); -const { version, srcDir, cleanup } = await resolveBenchmarkSource(); +const { srcDir, cleanup } = await resolveBenchmarkSource(); const dbPath = path.join(root, '.codegraph', 'graph.db'); const { buildGraph } = await import(srcImport(srcDir, 'builder.js')); -const { fnDepsData, fnImpactData, diffImpactData, statsData } = await import( +const { fnDepsData, fnImpactData, diffImpactData } = await import( srcImport(srcDir, 'queries.js') ); -const { isNativeAvailable } = await import( - srcImport(srcDir, 'native.js') -); -const { isWasmAvailable } = await import( - srcImport(srcDir, 'parser.js') -); // Redirect console.log to stderr so only JSON goes to stdout const origLog = console.log; console.log = (...args) => console.error(...args); const RUNS = 5; -const DEPTHS = [1, 3, 5]; function median(arr) { const sorted = [...arr].sort((a, b) => a - b); @@ -53,9 +79,6 @@ function round1(n) { return Math.round(n * 10) / 10; } -/** - * Select hub / mid / leaf targets dynamically from the graph. - */ function selectTargets() { const db = new Database(dbPath, { readonly: true }); const rows = db @@ -78,9 +101,6 @@ function selectTargets() { return { hub, mid, leaf }; } -/** - * Benchmark a single query function at multiple depths. - */ function benchDepths(fn, name, depths) { const result = {}; for (const depth of depths) { @@ -95,11 +115,7 @@ function benchDepths(fn, name, depths) { return result; } -/** - * Benchmark diff-impact with a synthetic staged change on the hub file. - */ function benchDiffImpact(hubName) { - // Find the file that contains the hub symbol const db = new Database(dbPath, { readonly: true }); const row = db .prepare(`SELECT file FROM nodes WHERE name = ? LIMIT 1`) @@ -112,7 +128,6 @@ function benchDiffImpact(hubName) { const original = fs.readFileSync(hubFile, 'utf8'); try { - // Append a probe comment and stage it fs.writeFileSync(hubFile, original + '\n// benchmark-probe\n'); execFileSync('git', ['add', hubFile], { cwd: root, stdio: 'pipe' }); @@ -130,95 +145,35 @@ function benchDiffImpact(hubName) { affectedFiles: lastResult?.affectedFiles?.length || 0, }; } finally { - // Restore: unstage + revert content execFileSync('git', ['restore', '--staged', hubFile], { cwd: root, stdio: 'pipe' }); fs.writeFileSync(hubFile, original); } } -/** - * Run all query benchmarks against the current graph. - */ -function benchmarkQueries(targets) { - const fnDeps = {}; - const fnImpact = {}; - - // Run depth benchmarks on hub target (most connected — worst case) - fnDeps.depth1Ms = benchDepths(fnDepsData, targets.hub, [1]).depth1Ms; - fnDeps.depth3Ms = benchDepths(fnDepsData, targets.hub, [3]).depth3Ms; - fnDeps.depth5Ms = benchDepths(fnDepsData, targets.hub, [5]).depth5Ms; - - fnImpact.depth1Ms = benchDepths(fnImpactData, targets.hub, [1]).depth1Ms; - fnImpact.depth3Ms = benchDepths(fnImpactData, targets.hub, [3]).depth3Ms; - fnImpact.depth5Ms = benchDepths(fnImpactData, targets.hub, [5]).depth5Ms; - - const diffImpact = benchDiffImpact(targets.hub); - - return { targets, fnDeps, fnImpact, diffImpact }; -} +// Build graph for this engine +if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); +await buildGraph(root, { engine, incremental: false }); -// ── Run benchmarks ─────────────────────────────────────────────────────── -const hasWasm = isWasmAvailable(); -const hasNative = isNativeAvailable(); +const targets = selectTargets(); +console.error(`Targets: hub=${targets.hub}, mid=${targets.mid}, leaf=${targets.leaf}`); -if (!hasWasm && !hasNative) { - console.error('Error: Neither WASM grammars nor native engine are available.'); - console.error('Run "npm run build:wasm" to build WASM grammars, or install the native platform package.'); - process.exit(1); -} +const fnDeps = {}; +const fnImpact = {}; -// Build with first available engine to select targets, then reuse for both -let targets = null; -let wasm = null; -if (hasWasm) { - if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); - await buildGraph(root, { engine: 'wasm', incremental: false }); - - targets = selectTargets(); - console.error(`Targets: hub=${targets.hub}, mid=${targets.mid}, leaf=${targets.leaf}`); - wasm = benchmarkQueries(targets); -} else { - console.error('WASM grammars not built — skipping WASM benchmark'); -} +fnDeps.depth1Ms = benchDepths(fnDepsData, targets.hub, [1]).depth1Ms; +fnDeps.depth3Ms = benchDepths(fnDepsData, targets.hub, [3]).depth3Ms; +fnDeps.depth5Ms = benchDepths(fnDepsData, targets.hub, [5]).depth5Ms; -let native = null; -if (hasNative) { - if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath); - await buildGraph(root, { engine: 'native', incremental: false }); +fnImpact.depth1Ms = benchDepths(fnImpactData, targets.hub, [1]).depth1Ms; +fnImpact.depth3Ms = benchDepths(fnImpactData, targets.hub, [3]).depth3Ms; +fnImpact.depth5Ms = benchDepths(fnImpactData, targets.hub, [5]).depth5Ms; - if (!targets) { - targets = selectTargets(); - console.error(`Targets: hub=${targets.hub}, mid=${targets.mid}, leaf=${targets.leaf}`); - } - native = benchmarkQueries(targets); -} else { - console.error('Native engine not available — skipping native benchmark'); -} +const diffImpact = benchDiffImpact(targets.hub); // Restore console.log for JSON output console.log = origLog; -const result = { - version, - date: new Date().toISOString().slice(0, 10), - wasm: wasm - ? { - targets: wasm.targets, - fnDeps: wasm.fnDeps, - fnImpact: wasm.fnImpact, - diffImpact: wasm.diffImpact, - } - : null, - native: native - ? { - targets: native.targets, - fnDeps: native.fnDeps, - fnImpact: native.fnImpact, - diffImpact: native.diffImpact, - } - : null, -}; - -console.log(JSON.stringify(result, null, 2)); +const workerResult = { targets, fnDeps, fnImpact, diffImpact }; +console.log(JSON.stringify(workerResult)); cleanup(); From fbea7293ff9c8a1c091a6588fee13fe722baa663 Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Wed, 18 Mar 2026 23:33:40 -0600 Subject: [PATCH 2/7] fix: extract shared forkWorker, remove dual timeout, add settle guard (#512) - Extract forkWorker() as a generic reusable subprocess helper - Remove fork()'s built-in timeout option so the manual SIGKILL timer is the sole timeout mechanism (they previously fired simultaneously) - Add a settled flag to prevent double Promise resolution on spawn failure - forkEngines() now delegates to forkWorker() instead of inline runWorker() --- scripts/lib/fork-engine.js | 137 ++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 63 deletions(-) diff --git a/scripts/lib/fork-engine.js b/scripts/lib/fork-engine.js index d0594777..b2d110d6 100644 --- a/scripts/lib/fork-engine.js +++ b/scripts/lib/fork-engine.js @@ -43,6 +43,78 @@ export function workerEngine() { return engine; } +/** + * Fork a single worker subprocess and collect its JSON output. + * + * @param {string} scriptPath Absolute path to the script to fork + * @param {string} envKey Environment variable name for the worker identifier + * @param {string} workerName Human-readable label for logging (e.g. 'wasm', 'gte-small') + * @param {string[]} argv CLI args to forward + * @param {number} [timeoutMs=600_000] Per-worker timeout (default 10 min) + * @returns {Promise} + */ +export function forkWorker(scriptPath, envKey, workerName, argv = [], timeoutMs = 600_000) { + return new Promise((resolve) => { + let settled = false; + function settle(value) { + if (settled) return; + settled = true; + resolve(value); + } + + console.error(`\n[fork] Spawning ${workerName} worker (pid isolation)...`); + + const child = fork(scriptPath, argv, { + env: { ...process.env, [envKey]: workerName }, + stdio: ['ignore', 'pipe', 'inherit', 'ipc'], + }); + + let stdout = ''; + child.stdout.on('data', (chunk) => { stdout += chunk; }); + + const timer = setTimeout(() => { + console.error(`[fork] ${workerName} worker timed out after ${timeoutMs / 1000}s — killing`); + child.kill('SIGKILL'); + }, timeoutMs); + + child.on('close', (code, signal) => { + clearTimeout(timer); + + if (signal) { + console.error(`[fork] ${workerName} worker killed by signal ${signal}`); + settle(null); + return; + } + + if (code !== 0) { + console.error(`[fork] ${workerName} worker exited with code ${code}`); + // Try to parse partial output anyway + try { + const parsed = JSON.parse(stdout); + console.error(`[fork] ${workerName} worker produced partial results despite non-zero exit`); + settle(parsed); + } catch { + settle(null); + } + return; + } + + try { + settle(JSON.parse(stdout)); + } catch (err) { + console.error(`[fork] ${workerName} worker produced invalid JSON: ${err.message}`); + settle(null); + } + }); + + child.on('error', (err) => { + clearTimeout(timer); + console.error(`[fork] ${workerName} worker failed to start: ${err.message}`); + settle(null); + }); + }); +} + /** * Fork the calling script once per available engine, collect JSON results. * @@ -83,78 +155,17 @@ export async function forkEngines(scriptUrl, argv = [], opts = {}) { process.exit(1); } - /** - * Fork a single engine worker and collect its JSON output. - * @param {string} engine - * @returns {Promise} - */ - function runWorker(engine) { - return new Promise((resolve) => { - console.error(`\n[fork] Spawning ${engine} worker (pid isolation)...`); - - const child = fork(scriptPath, argv, { - env: { ...process.env, [WORKER_ENV_KEY]: engine }, - stdio: ['ignore', 'pipe', 'inherit', 'ipc'], - timeout: timeoutMs, - }); - - let stdout = ''; - child.stdout.on('data', (chunk) => { stdout += chunk; }); - - const timer = setTimeout(() => { - console.error(`[fork] ${engine} worker timed out after ${timeoutMs / 1000}s — killing`); - child.kill('SIGKILL'); - }, timeoutMs); - - child.on('close', (code, signal) => { - clearTimeout(timer); - - if (signal) { - console.error(`[fork] ${engine} worker killed by signal ${signal}`); - resolve(null); - return; - } - - if (code !== 0) { - console.error(`[fork] ${engine} worker exited with code ${code}`); - // Try to parse partial output anyway - try { - const parsed = JSON.parse(stdout); - console.error(`[fork] ${engine} worker produced partial results despite non-zero exit`); - resolve(parsed); - } catch { - resolve(null); - } - return; - } - - try { - resolve(JSON.parse(stdout)); - } catch (err) { - console.error(`[fork] ${engine} worker produced invalid JSON: ${err.message}`); - resolve(null); - } - }); - - child.on('error', (err) => { - clearTimeout(timer); - console.error(`[fork] ${engine} worker failed to start: ${err.message}`); - resolve(null); - }); - }); - } - const results = { wasm: null, native: null }; // Run engines sequentially — they share the DB file and filesystem state. if (hasWasm) { - results.wasm = await runWorker('wasm'); + results.wasm = await forkWorker(scriptPath, WORKER_ENV_KEY, 'wasm', argv, timeoutMs); } else { console.error('WASM grammars not built — skipping WASM benchmark'); } if (hasNative) { - results.native = await runWorker('native'); + results.native = await forkWorker(scriptPath, WORKER_ENV_KEY, 'native', argv, timeoutMs); } else { console.error('Native engine not available — skipping native benchmark'); } From 24e2fb0a62f49e0f6b78d909425537acbd0177ae Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Wed, 18 Mar 2026 23:33:49 -0600 Subject: [PATCH 3/7] fix: capture and call cleanup from resolveBenchmarkSource in parent (#512) The first resolveBenchmarkSource() call (for version extraction) never had its cleanup invoked, potentially leaking temporary resources. --- scripts/benchmark.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/benchmark.js b/scripts/benchmark.js index c2651443..9fb135ac 100644 --- a/scripts/benchmark.js +++ b/scripts/benchmark.js @@ -20,7 +20,7 @@ import { isWorker, workerEngine, forkEngines } from './lib/fork-engine.js'; // ── Parent process: fork one child per engine, assemble final output ───── if (!isWorker()) { - const { version } = await resolveBenchmarkSource(); + const { version, cleanup: versionCleanup } = await resolveBenchmarkSource(); const { wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2)); const primary = wasm || native; @@ -66,6 +66,7 @@ if (!isWorker()) { }; console.log(JSON.stringify(result, null, 2)); + versionCleanup(); process.exit(0); } From 3aa0348c0cef329a66fc218b9ce9f593924744d9 Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Wed, 18 Mar 2026 23:33:57 -0600 Subject: [PATCH 4/7] fix: replace duplicated forkModel with shared forkWorker (#512) embedding-benchmark.js had a near-identical copy of runWorker() from fork-engine.js (including the dual-timeout bug). Now imports and uses the shared forkWorker() helper instead. --- scripts/embedding-benchmark.js | 59 ++-------------------------------- 1 file changed, 2 insertions(+), 57 deletions(-) diff --git a/scripts/embedding-benchmark.js b/scripts/embedding-benchmark.js index 35344011..714cdb3e 100644 --- a/scripts/embedding-benchmark.js +++ b/scripts/embedding-benchmark.js @@ -10,12 +10,12 @@ * Usage: node scripts/embedding-benchmark.js > result.json */ -import { fork } from 'node:child_process'; import path from 'node:path'; import { performance } from 'node:perf_hooks'; import { fileURLToPath } from 'node:url'; import Database from 'better-sqlite3'; import { resolveBenchmarkSource, srcImport } from './lib/bench-config.js'; +import { forkWorker } from './lib/fork-engine.js'; const MODEL_WORKER_KEY = '__BENCH_MODEL__'; @@ -133,68 +133,13 @@ let symbolCount = 0; const scriptPath = fileURLToPath(import.meta.url); -function forkModel(modelKey) { - return new Promise((resolve) => { - console.error(`\n[fork] Spawning ${modelKey} worker (pid isolation)...`); - - const child = fork(scriptPath, process.argv.slice(2), { - env: { ...process.env, [MODEL_WORKER_KEY]: modelKey }, - stdio: ['ignore', 'pipe', 'inherit', 'ipc'], - timeout: TIMEOUT_MS, - }); - - let stdout = ''; - child.stdout.on('data', (chunk) => { stdout += chunk; }); - - const timer = setTimeout(() => { - console.error(`[fork] ${modelKey} worker timed out after ${TIMEOUT_MS / 1000}s — killing`); - child.kill('SIGKILL'); - }, TIMEOUT_MS); - - child.on('close', (code, signal) => { - clearTimeout(timer); - - if (signal) { - console.error(`[fork] ${modelKey} worker killed by signal ${signal}`); - resolve(null); - return; - } - - if (code !== 0) { - console.error(`[fork] ${modelKey} worker exited with code ${code}`); - try { - const parsed = JSON.parse(stdout); - console.error(`[fork] ${modelKey} worker produced partial results despite non-zero exit`); - resolve(parsed); - } catch { - resolve(null); - } - return; - } - - try { - resolve(JSON.parse(stdout)); - } catch (err) { - console.error(`[fork] ${modelKey} worker produced invalid JSON: ${err.message}`); - resolve(null); - } - }); - - child.on('error', (err) => { - clearTimeout(timer); - console.error(`[fork] ${modelKey} worker failed to start: ${err.message}`); - resolve(null); - }); - }); -} - for (const key of modelKeys) { if (key === 'jina-code' && !hasHfToken) { console.error(`Skipping ${key} (HF_TOKEN not set)`); continue; } - const data = await forkModel(key); + const data = await forkWorker(scriptPath, MODEL_WORKER_KEY, key, process.argv.slice(2), TIMEOUT_MS); if (data) { results[key] = data.result; if (data.symbols) symbolCount = data.symbols; From 1504b42bb6e978245879b28c960939ba97cbf92e Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Wed, 18 Mar 2026 23:42:55 -0600 Subject: [PATCH 5/7] fix: call versionCleanup on error exit path in benchmark.js (#512) versionCleanup() was only called on the success path. When both engines fail and the parent exits early via process.exit(1), temporary resources from resolveBenchmarkSource() were leaked. --- scripts/benchmark.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/benchmark.js b/scripts/benchmark.js index 9fb135ac..b12daeab 100644 --- a/scripts/benchmark.js +++ b/scripts/benchmark.js @@ -26,6 +26,7 @@ if (!isWorker()) { const primary = wasm || native; if (!primary) { console.error('Error: Both engines failed. No results to report.'); + versionCleanup(); process.exit(1); } From 0e7fa61613ba310b6194b1b3b93ceac2b79fb534 Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Wed, 18 Mar 2026 23:42:59 -0600 Subject: [PATCH 6/7] fix: add parent cleanup and git safety net in query-benchmark.js (#512) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two issues addressed: 1. resolveBenchmarkSource() cleanup was never captured or called in the parent process, leaking temporary resources on every run. 2. benchDiffImpact() runs git-add inside the worker — if a segfault kills the worker mid-execution, the finally block never runs and the git staging area is left dirty. The parent now checks for leftover staged files after workers exit and cleans them up. --- scripts/query-benchmark.js | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/scripts/query-benchmark.js b/scripts/query-benchmark.js index 0360040c..3ab93274 100644 --- a/scripts/query-benchmark.js +++ b/scripts/query-benchmark.js @@ -21,9 +21,33 @@ import { isWorker, workerEngine, forkEngines } from './lib/fork-engine.js'; // ── Parent process: fork one child per engine, assemble final output ───── if (!isWorker()) { - const { version } = await resolveBenchmarkSource(); + const __parentDir = path.dirname(fileURLToPath(import.meta.url)); + const __parentRoot = path.resolve(__parentDir, '..'); + + const { version, cleanup: versionCleanup } = await resolveBenchmarkSource(); const { wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2)); + // Safety net: if a worker was killed mid-benchDiffImpact, the git staging + // area may be dirty. Unstage any leftover changes so subsequent runs and + // unrelated git operations aren't affected. + try { + const staged = execFileSync('git', ['diff', '--cached', '--name-only'], { + cwd: __parentRoot, encoding: 'utf8', + }).trim(); + if (staged) { + console.error('[fork] Cleaning up leftover staged files from crashed worker'); + execFileSync('git', ['restore', '--staged', '.'], { cwd: __parentRoot, stdio: 'pipe' }); + execFileSync('git', ['checkout', '.'], { cwd: __parentRoot, stdio: 'pipe' }); + } + } catch { /* git not available or no repo — safe to ignore */ } + + const primary = wasm || native; + if (!primary) { + console.error('Error: Both engines failed. No results to report.'); + versionCleanup(); + process.exit(1); + } + const result = { version, date: new Date().toISOString().slice(0, 10), @@ -46,6 +70,7 @@ if (!isWorker()) { }; console.log(JSON.stringify(result, null, 2)); + versionCleanup(); process.exit(0); } From eb93ddc0a2c4abf16a1d3660e25eeab5a09b1495 Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Wed, 18 Mar 2026 23:59:09 -0600 Subject: [PATCH 7/7] fix: replace process.exit in forkEngines with thrown error (#512) forkEngines() called process.exit(1) when no engines were available, which aborted the call-stack immediately and prevented callers from invoking their own cleanup callbacks (versionCleanup, parentCleanup). Now throws an Error instead; all three callers catch it, run cleanup, and exit gracefully. Impact: 1 functions changed, 3 affected --- scripts/benchmark.js | 9 ++++++++- scripts/incremental-benchmark.js | 9 ++++++++- scripts/lib/fork-engine.js | 6 +++--- scripts/query-benchmark.js | 9 ++++++++- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/scripts/benchmark.js b/scripts/benchmark.js index b12daeab..c2202f31 100644 --- a/scripts/benchmark.js +++ b/scripts/benchmark.js @@ -21,7 +21,14 @@ import { isWorker, workerEngine, forkEngines } from './lib/fork-engine.js'; // ── Parent process: fork one child per engine, assemble final output ───── if (!isWorker()) { const { version, cleanup: versionCleanup } = await resolveBenchmarkSource(); - const { wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2)); + let wasm, native; + try { + ({ wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2))); + } catch (err) { + console.error(`Error: ${err.message}`); + versionCleanup(); + process.exit(1); + } const primary = wasm || native; if (!primary) { diff --git a/scripts/incremental-benchmark.js b/scripts/incremental-benchmark.js index 94c3ac9b..93c71993 100644 --- a/scripts/incremental-benchmark.js +++ b/scripts/incremental-benchmark.js @@ -20,7 +20,14 @@ import { isWorker, workerEngine, forkEngines } from './lib/fork-engine.js'; // ── Parent process: fork one child per engine, assemble final output ───── if (!isWorker()) { const { version, srcDir: parentSrcDir, cleanup: parentCleanup } = await resolveBenchmarkSource(); - const { wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2)); + let wasm, native; + try { + ({ wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2))); + } catch (err) { + console.error(`Error: ${err.message}`); + parentCleanup(); + process.exit(1); + } // Import resolution runs in the parent — it tests both native and JS // fallback in a single pass and doesn't need engine isolation. diff --git a/scripts/lib/fork-engine.js b/scripts/lib/fork-engine.js index b2d110d6..97a6880a 100644 --- a/scripts/lib/fork-engine.js +++ b/scripts/lib/fork-engine.js @@ -150,9 +150,9 @@ export async function forkEngines(scriptUrl, argv = [], opts = {}) { cleanup(); if (!hasWasm && !hasNative) { - console.error('Error: Neither WASM grammars nor native engine are available.'); - console.error('Run "npm run build:wasm" to build WASM grammars, or install the native platform package.'); - process.exit(1); + const msg = 'Neither WASM grammars nor native engine are available. ' + + 'Run "npm run build:wasm" to build WASM grammars, or install the native platform package.'; + throw new Error(msg); } const results = { wasm: null, native: null }; diff --git a/scripts/query-benchmark.js b/scripts/query-benchmark.js index 3ab93274..6fbe14a4 100644 --- a/scripts/query-benchmark.js +++ b/scripts/query-benchmark.js @@ -25,7 +25,14 @@ if (!isWorker()) { const __parentRoot = path.resolve(__parentDir, '..'); const { version, cleanup: versionCleanup } = await resolveBenchmarkSource(); - const { wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2)); + let wasm, native; + try { + ({ wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2))); + } catch (err) { + console.error(`Error: ${err.message}`); + versionCleanup(); + process.exit(1); + } // Safety net: if a worker was killed mid-benchDiffImpact, the git staging // area may be dirty. Unstage any leftover changes so subsequent runs and