Skip to content

feat: universal pagination, NDJSON streaming, and generator APIs #207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
carlos-alm merged 2 commits into from
Mar 2, 2026
Merged

feat: universal pagination, NDJSON streaming, and generator APIs #207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
30a99a9
feat: universal pagination, NDJSON streaming, and generator APIs
carlos-alm Mar 2, 2026
c930573
perf(queries): hoist prepared statements out of iterWhere loop
carlos-alm Mar 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 92 additions & 6 deletions src/cli.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
} from './embedder.js';
import { exportDOT, exportJSON, exportMermaid } from './export.js';
import { setVerbose } from './logger.js';
import { printNdjson } from './paginate.js';
import {
ALL_SYMBOL_KINDS,
context,
Expand Down Expand Up @@ -127,8 +128,17 @@ program
.option('-T, --no-tests', 'Exclude test/spec files from results')
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('--limit <number>', 'Max results to return')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action((file, opts) => {
impactAnalysis(file, opts.db, { noTests: resolveNoTests(opts), json: opts.json });
impactAnalysis(file, opts.db, {
noTests: resolveNoTests(opts),
json: opts.json,
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
ndjson: opts.ndjson,
});
});

program
Expand Down Expand Up @@ -164,8 +174,17 @@ program
.option('-T, --no-tests', 'Exclude test/spec files from results')
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('--limit <number>', 'Max results to return')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action((file, opts) => {
fileDeps(file, opts.db, { noTests: resolveNoTests(opts), json: opts.json });
fileDeps(file, opts.db, {
noTests: resolveNoTests(opts),
json: opts.json,
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
ndjson: opts.ndjson,
});
});

program
Expand All @@ -178,6 +197,9 @@ program
.option('-T, --no-tests', 'Exclude test/spec files from results')
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('--limit <number>', 'Max results to return')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action((name, opts) => {
if (opts.kind && !ALL_SYMBOL_KINDS.includes(opts.kind)) {
console.error(`Invalid kind "${opts.kind}". Valid: ${ALL_SYMBOL_KINDS.join(', ')}`);
Expand All @@ -189,6 +211,9 @@ program
kind: opts.kind,
noTests: resolveNoTests(opts),
json: opts.json,
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
ndjson: opts.ndjson,
});
});

Expand All @@ -202,6 +227,9 @@ program
.option('-T, --no-tests', 'Exclude test/spec files from results')
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('--limit <number>', 'Max results to return')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action((name, opts) => {
if (opts.kind && !ALL_SYMBOL_KINDS.includes(opts.kind)) {
console.error(`Invalid kind "${opts.kind}". Valid: ${ALL_SYMBOL_KINDS.join(', ')}`);
Expand All @@ -213,6 +241,9 @@ program
kind: opts.kind,
noTests: resolveNoTests(opts),
json: opts.json,
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
ndjson: opts.ndjson,
});
});

Expand Down Expand Up @@ -258,6 +289,9 @@ program
.option('-T, --no-tests', 'Exclude test/spec files from results')
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('--limit <number>', 'Max results to return')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action((name, opts) => {
if (opts.kind && !ALL_SYMBOL_KINDS.includes(opts.kind)) {
console.error(`Invalid kind "${opts.kind}". Valid: ${ALL_SYMBOL_KINDS.join(', ')}`);
Expand All @@ -271,6 +305,9 @@ program
noTests: resolveNoTests(opts),
includeTests: opts.withTestSource,
json: opts.json,
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
ndjson: opts.ndjson,
});
});

Expand All @@ -282,11 +319,17 @@ program
.option('-T, --no-tests', 'Exclude test/spec files from results')
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('--limit <number>', 'Max results to return')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action((target, opts) => {
explain(target, opts.db, {
depth: parseInt(opts.depth, 10),
noTests: resolveNoTests(opts),
json: opts.json,
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
ndjson: opts.ndjson,
});
});

Expand Down Expand Up @@ -327,6 +370,9 @@ program
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('-f, --format <format>', 'Output format: text, mermaid, json', 'text')
.option('--limit <number>', 'Max results to return')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action((ref, opts) => {
diffImpact(opts.db, {
ref,
Expand All @@ -335,6 +381,9 @@ program
noTests: resolveNoTests(opts),
json: opts.json,
format: opts.format,
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
ndjson: opts.ndjson,
});
});

Expand Down Expand Up @@ -640,6 +689,8 @@ program
.option('--rrf-k <number>', 'RRF k parameter for multi-query ranking', '60')
.option('--mode <mode>', 'Search mode: hybrid, semantic, keyword (default: hybrid)')
.option('-j, --json', 'Output as JSON')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action(async (query, opts) => {
const validModes = ['hybrid', 'semantic', 'keyword'];
if (opts.mode && !validModes.includes(opts.mode)) {
Expand Down Expand Up @@ -671,6 +722,9 @@ program
.option('-T, --no-tests', 'Exclude test/spec files')
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('--limit <number>', 'Max results to return')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action(async (dir, opts) => {
const { structureData, formatStructure } = await import('./structure.js');
const data = structureData(opts.db, {
Expand All @@ -679,8 +733,12 @@ program
sort: opts.sort,
full: opts.full,
noTests: resolveNoTests(opts),
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
});
if (opts.json) {
if (opts.ndjson) {
printNdjson(data, 'directories');
} else if (opts.json) {
console.log(JSON.stringify(data, null, 2));
} else {
console.log(formatStructure(data));
Expand All @@ -699,15 +757,20 @@ program
.option('-T, --no-tests', 'Exclude test/spec files from results')
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action(async (opts) => {
const { hotspotsData, formatHotspots } = await import('./structure.js');
const data = hotspotsData(opts.db, {
metric: opts.metric,
level: opts.level,
limit: parseInt(opts.limit, 10),
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
noTests: resolveNoTests(opts),
});
if (opts.json) {
if (opts.ndjson) {
printNdjson(data, 'hotspots');
} else if (opts.json) {
console.log(JSON.stringify(data, null, 2));
} else {
console.log(formatHotspots(data));
Expand Down Expand Up @@ -757,6 +820,8 @@ program
.option('-T, --no-tests', 'Exclude test/spec files')
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action(async (file, opts) => {
const { analyzeCoChanges, coChangeData, coChangeTopData, formatCoChange, formatCoChangeTop } =
await import('./cochange.js');
Expand All @@ -783,20 +848,25 @@ program

const queryOpts = {
limit: parseInt(opts.limit, 10),
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
minJaccard: opts.minJaccard ? parseFloat(opts.minJaccard) : config.coChange?.minJaccard,
noTests: resolveNoTests(opts),
};

if (file) {
const data = coChangeData(file, opts.db, queryOpts);
if (opts.json) {
if (opts.ndjson) {
printNdjson(data, 'partners');
} else if (opts.json) {
console.log(JSON.stringify(data, null, 2));
} else {
console.log(formatCoChange(data));
}
} else {
const data = coChangeTopData(opts.db, queryOpts);
if (opts.json) {
if (opts.ndjson) {
printNdjson(data, 'pairs');
} else if (opts.json) {
console.log(JSON.stringify(data, null, 2));
} else {
console.log(formatCoChangeTop(data));
Expand Down Expand Up @@ -860,6 +930,8 @@ program
.option('-T, --no-tests', 'Exclude test/spec files from results')
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action(async (target, opts) => {
if (opts.kind && !ALL_SYMBOL_KINDS.includes(opts.kind)) {
console.error(`Invalid kind "${opts.kind}". Valid: ${ALL_SYMBOL_KINDS.join(', ')}`);
Expand All @@ -869,13 +941,15 @@ program
complexity(opts.db, {
target,
limit: parseInt(opts.limit, 10),
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
sort: opts.sort,
aboveThreshold: opts.aboveThreshold,
health: opts.health,
file: opts.file,
kind: opts.kind,
noTests: resolveNoTests(opts),
json: opts.json,
ndjson: opts.ndjson,
});
});

Expand All @@ -888,6 +962,9 @@ program
.option('-f, --file <path>', 'Scope to file (partial match)')
.option('-k, --kind <kind>', 'Filter by symbol kind')
.option('-j, --json', 'Output as JSON')
.option('--limit <number>', 'Max results to return')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action(async (opts) => {
if (opts.kind && !ALL_SYMBOL_KINDS.includes(opts.kind)) {
console.error(`Invalid kind "${opts.kind}". Valid: ${ALL_SYMBOL_KINDS.join(', ')}`);
Expand All @@ -899,6 +976,9 @@ program
kind: opts.kind,
noTests: resolveNoTests(opts),
json: opts.json,
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
ndjson: opts.ndjson,
});
});

Expand All @@ -912,6 +992,9 @@ program
.option('-T, --no-tests', 'Exclude test/spec files from results')
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
.option('-j, --json', 'Output as JSON')
.option('--limit <number>', 'Max results to return')
.option('--offset <number>', 'Skip N results (default: 0)')
.option('--ndjson', 'Newline-delimited JSON output')
.action(async (opts) => {
const { communities } = await import('./communities.js');
communities(opts.db, {
Expand All @@ -920,6 +1003,9 @@ program
drift: opts.drift,
noTests: resolveNoTests(opts),
json: opts.json,
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
ndjson: opts.ndjson,
});
});

Expand Down
7 changes: 5 additions & 2 deletions src/cochange.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import path from 'node:path';
import { normalizePath } from './constants.js';
import { closeDb, findDbPath, initSchema, openDb, openReadonlyOrFail } from './db.js';
import { warn } from './logger.js';
import { paginateResult } from './paginate.js';
import { isTestFile } from './queries.js';

/**
Expand Down Expand Up @@ -313,7 +314,8 @@ export function coChangeData(file, customDbPath, opts = {}) {
const meta = getCoChangeMeta(db);
closeDb(db);

return { file: resolvedFile, partners, meta };
const base = { file: resolvedFile, partners, meta };
return paginateResult(base, 'partners', { limit: opts.limit, offset: opts.offset });
}

/**
Expand Down Expand Up @@ -365,7 +367,8 @@ export function coChangeTopData(customDbPath, opts = {}) {
const meta = getCoChangeMeta(db);
closeDb(db);

return { pairs, meta };
const base = { pairs, meta };
return paginateResult(base, 'pairs', { limit: opts.limit, offset: opts.offset });
}

/**
Expand Down
8 changes: 7 additions & 1 deletion src/communities.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import path from 'node:path';
import Graph from 'graphology';
import louvain from 'graphology-communities-louvain';
import { openReadonlyOrFail } from './db.js';
import { paginateResult, printNdjson } from './paginate.js';
import { isTestFile } from './queries.js';

// ─── Graph Construction ───────────────────────────────────────────────
Expand Down Expand Up @@ -201,7 +202,7 @@ export function communitiesData(customDbPath, opts = {}) {

const driftScore = Math.round(((splitRatio + mergeRatio) / 2) * 100);

return {
const base = {
communities: opts.drift ? [] : communities,
modularity: +modularity.toFixed(4),
drift: { splitCandidates, mergeCandidates },
Expand All @@ -212,6 +213,7 @@ export function communitiesData(customDbPath, opts = {}) {
driftScore,
},
};
return paginateResult(base, 'communities', { limit: opts.limit, offset: opts.offset });
}

/**
Expand All @@ -238,6 +240,10 @@ export function communitySummaryForStats(customDbPath, opts = {}) {
export function communities(customDbPath, opts = {}) {
const data = communitiesData(customDbPath, opts);

if (opts.ndjson) {
printNdjson(data, 'communities');
return;
}
if (opts.json) {
console.log(JSON.stringify(data, null, 2));
return;
Expand Down
Loading