diff --git a/src/server/core/domain/entities/query-log-entry.ts b/src/server/core/domain/entities/query-log-entry.ts index 67045a55f..2b183f19c 100644 --- a/src/server/core/domain/entities/query-log-entry.ts +++ b/src/server/core/domain/entities/query-log-entry.ts @@ -6,6 +6,8 @@ export const QUERY_LOG_TIERS = [0, 1, 2, 3, 4] as const export type QueryLogTier = (typeof QUERY_LOG_TIERS)[number] +export type TierKey = `tier${QueryLogTier}` + /** Human-readable labels for each resolution tier. */ export const QUERY_LOG_TIER_LABELS: Record = { 0: 'exact cache hit', @@ -15,6 +17,22 @@ export const QUERY_LOG_TIER_LABELS: Record = { 4: 'full agentic', } +/** Tiers considered cache hits for cache-hit-rate calculation. */ +export const CACHE_TIERS = [0, 1] as const satisfies readonly QueryLogTier[] + +export type ByTier = Record & {unknown: number} + +// Single `as` contained here — TS cannot prove loop exhaustiveness over template literal keys. +export function emptyByTier(): ByTier { + const obj: Record = {} + for (const t of QUERY_LOG_TIERS) { + obj[`tier${t}`] = 0 + } + + obj.unknown = 0 + return obj as ByTier +} + /** Valid query log statuses. Add/remove here — the type updates automatically. */ export const QUERY_LOG_STATUSES = ['cancelled', 'completed', 'error', 'processing'] as const export type QueryLogStatus = (typeof QUERY_LOG_STATUSES)[number] diff --git a/src/server/core/interfaces/usecase/i-query-log-summary-use-case.ts b/src/server/core/interfaces/usecase/i-query-log-summary-use-case.ts index bd3b38273..10b2dc30c 100644 --- a/src/server/core/interfaces/usecase/i-query-log-summary-use-case.ts +++ b/src/server/core/interfaces/usecase/i-query-log-summary-use-case.ts @@ -1,5 +1,4 @@ -// Interface driven by `brv query-log summary` command (ENG-1899). -// Full implementation of data computation, text format, and JSON format in ENG-1898. +import type {ByTier} from '../../domain/entities/query-log-entry.js' export type QueryLogSummary = { byStatus: { @@ -7,14 +6,7 @@ export type QueryLogSummary = { completed: number error: number } - byTier: { - tier0: number - tier1: number - tier2: number - tier3: number - tier4: number - unknown: number - } + byTier: ByTier cacheHitRate: number coverageRate: number knowledgeGaps: { diff --git a/src/server/infra/usecase/query-log-summary-narrative-formatter.ts b/src/server/infra/usecase/query-log-summary-narrative-formatter.ts index a30822107..6a8cc4744 100644 --- a/src/server/infra/usecase/query-log-summary-narrative-formatter.ts +++ b/src/server/infra/usecase/query-log-summary-narrative-formatter.ts @@ -1,26 +1,20 @@ import type {QueryLogSummary} from '../../core/interfaces/usecase/i-query-log-summary-use-case.js' -const EMPTY_STATE_MESSAGE = - 'No queries recorded in the last 24 hours. Your knowledge base is ready — try asking a question!' - const NARRATIVE_TOP_DOCS = 2 const NARRATIVE_TOP_GAPS = 2 +const MS_PER_HOUR = 3_600_000 +const MS_PER_DAY = 86_400_000 -/** - * Format a QueryLogSummary as a human-readable value story. - * - * Pure formatting — no computation. Consumes the data already computed - * by QueryLogSummaryUseCase (ENG-1898) and wraps it in prose aimed at - * conveying ByteRover recall value to end users. - */ export function formatQueryLogSummaryNarrative(summary: QueryLogSummary): string { + const periodLabel = describePeriod(summary.period) + if (summary.totalQueries === 0) { - return EMPTY_STATE_MESSAGE + return `No queries recorded ${periodLabel}. Your knowledge base is ready — try asking a question!` } const paragraphs: string[] = [] - paragraphs.push(buildOverviewParagraph(summary)) + paragraphs.push(buildOverviewParagraph(summary, periodLabel)) if (summary.totalMatchedDocs > 0 && summary.topRecalledDocs.length > 0) { paragraphs.push(buildTopDocsParagraph(summary)) @@ -31,14 +25,41 @@ export function formatQueryLogSummaryNarrative(summary: QueryLogSummary): string return paragraphs.join('\n\n') } -function buildOverviewParagraph(summary: QueryLogSummary): string { - const {cacheHitRate, coverageRate, queriesWithoutMatches, responseTime, totalQueries} = summary - const answered = totalQueries - queriesWithoutMatches +function describePeriod(period: QueryLogSummary['period']): string { + if (period.from === 0 && period.to === 0) { + return 'in the selected period' + } + + if (period.from > 0 && period.to > 0) { + const spanMs = period.to - period.from + return describeSpan(spanMs) + } + + if (period.from > 0) { + const spanMs = Date.now() - period.from + return describeSpan(spanMs) + } + + return 'in the selected period' +} + +function describeSpan(spanMs: number): string { + if (spanMs <= MS_PER_DAY + MS_PER_HOUR) { + return 'in the last 24 hours' + } + + const days = Math.round(spanMs / MS_PER_DAY) + return `in the last ${days} days` +} + +function buildOverviewParagraph(summary: QueryLogSummary, periodLabel: string): string { + const {byStatus, cacheHitRate, coverageRate, queriesWithoutMatches, responseTime, totalQueries} = summary + const answered = byStatus.completed - queriesWithoutMatches const coveragePct = Math.round(coverageRate * 100) const cachePct = Math.round(cacheHitRate * 100) return ( - `Your team asked ${totalQueries} questions today. ` + + `Your team asked ${totalQueries} questions ${periodLabel}. ` + `ByteRover answered ${answered} from curated knowledge ` + `(${coveragePct}% coverage), with ${cachePct}% served from cache. ` + `Average response time was ${formatDuration(responseTime.avgMs)}.` diff --git a/src/server/infra/usecase/query-log-summary-use-case.ts b/src/server/infra/usecase/query-log-summary-use-case.ts index 395c4d74f..e99ec7090 100644 --- a/src/server/infra/usecase/query-log-summary-use-case.ts +++ b/src/server/infra/usecase/query-log-summary-use-case.ts @@ -1,11 +1,12 @@ +import type {QueryLogEntry} from '../../core/domain/entities/query-log-entry.js' import type {ITerminal} from '../../core/interfaces/i-terminal.js' import type {IQueryLogStore} from '../../core/interfaces/storage/i-query-log-store.js' import type { IQueryLogSummaryUseCase, QueryLogSummary, - QueryLogSummaryFormat, } from '../../core/interfaces/usecase/i-query-log-summary-use-case.js' +import {CACHE_TIERS, emptyByTier, QUERY_LOG_TIER_LABELS, QUERY_LOG_TIERS} from '../../core/domain/entities/query-log-entry.js' import {formatQueryLogSummaryNarrative} from './query-log-summary-narrative-formatter.js' type QueryLogSummaryUseCaseDeps = { @@ -14,13 +15,57 @@ type QueryLogSummaryUseCaseDeps = { } const EMPTY_TEXT_OUTPUT = 'Query Recall Summary\n(no entries yet)' +const TOP_LIMIT = 10 +const MAX_EXAMPLE_QUERIES = 3 +const MIN_KEYWORD_LENGTH = 3 + +const STOPWORDS = new Set([ + 'about', + 'all', + 'and', + 'any', + 'are', + 'but', + 'can', + 'did', + 'does', + 'for', + 'from', + 'get', + 'had', + 'has', + 'have', + 'how', + 'into', + 'just', + 'like', + 'not', + 'our', + 'set', + 'that', + 'the', + 'this', + 'use', + 'using', + 'was', + 'were', + 'what', + 'when', + 'where', + 'which', + 'who', + 'why', + 'with', + 'you', + 'your', +]) export class QueryLogSummaryUseCase implements IQueryLogSummaryUseCase { constructor(private readonly deps: QueryLogSummaryUseCaseDeps) {} - async run(options: {after?: number; before?: number; format?: QueryLogSummaryFormat}): Promise { - // TODO(ENG-1898): replace makeZeroSummary() with real aggregation from this.deps.queryLogStore - const summary = makeZeroSummary() + async run(options: Parameters[0]): Promise { + const entries = await this.deps.queryLogStore.list({after: options.after, before: options.before}) + const summary = computeSummary(entries, {after: options.after, before: options.before}) const format = options.format ?? 'text' if (format === 'narrative') { @@ -33,18 +78,20 @@ export class QueryLogSummaryUseCase implements IQueryLogSummaryUseCase { return } - this.deps.terminal.log(EMPTY_TEXT_OUTPUT) + this.deps.terminal.log(formatSummaryText(summary)) } } -function makeZeroSummary(): QueryLogSummary { - return { +// ── Aggregation ───────────────────────────────────────────────────────────── + +function computeSummary(entries: QueryLogEntry[], range: {after?: number; before?: number}): QueryLogSummary { + const summary: QueryLogSummary = { byStatus: {cancelled: 0, completed: 0, error: 0}, - byTier: {tier0: 0, tier1: 0, tier2: 0, tier3: 0, tier4: 0, unknown: 0}, + byTier: emptyByTier(), cacheHitRate: 0, coverageRate: 0, knowledgeGaps: [], - period: {from: 0, to: 0}, + period: {from: range.after ?? 0, to: range.before ?? 0}, queriesWithoutMatches: 0, responseTime: {avgMs: 0, p50Ms: 0, p95Ms: 0}, topRecalledDocs: [], @@ -52,4 +99,183 @@ function makeZeroSummary(): QueryLogSummary { totalMatchedDocs: 0, totalQueries: 0, } + + if (entries.length === 0) { + return summary + } + + const durations: number[] = [] + const topicCounts = new Map() + const docCounts = new Map() + const gapBuckets = new Map() + let completedWithMatches = 0 + + for (const entry of entries) { + if (entry.status === 'processing') continue + + summary.totalQueries += 1 + summary.byStatus[entry.status] += 1 + + if (entry.status !== 'completed') continue + + // ── completed-only aggregations ── + if (entry.timing) { + durations.push(entry.timing.durationMs) + } + + if (entry.tier === undefined) { + summary.byTier.unknown += 1 + } else { + summary.byTier[`tier${entry.tier}`] += 1 + } + + summary.totalMatchedDocs += entry.matchedDocs.length + + if (entry.matchedDocs.length > 0) { + completedWithMatches += 1 + for (const doc of entry.matchedDocs) { + const topic = doc.path.split('/')[0] + topicCounts.set(topic, (topicCounts.get(topic) ?? 0) + 1) + docCounts.set(doc.path, (docCounts.get(doc.path) ?? 0) + 1) + } + } else { + collectGapKeywords(entry.query, gapBuckets) + } + } + + if (summary.byStatus.completed > 0) { + const cacheHits = CACHE_TIERS.reduce((sum, t) => sum + summary.byTier[`tier${t}`], 0) + summary.cacheHitRate = cacheHits / summary.byStatus.completed + summary.coverageRate = completedWithMatches / summary.byStatus.completed + summary.queriesWithoutMatches = summary.byStatus.completed - completedWithMatches + } + + summary.responseTime = computeResponseTime(durations) + summary.topTopics = sortAndLimitCounts(topicCounts, 'topic') + summary.topRecalledDocs = sortAndLimitCounts(docCounts, 'path') + summary.knowledgeGaps = sortAndLimitGaps(gapBuckets) + + return summary +} + +function computeResponseTime(durations: number[]): QueryLogSummary['responseTime'] { + if (durations.length === 0) { + return {avgMs: 0, p50Ms: 0, p95Ms: 0} + } + + const sorted = [...durations].sort((a, b) => a - b) + const sum = sorted.reduce((acc, v) => acc + v, 0) + return { + avgMs: Math.round(sum / sorted.length), + p50Ms: sorted[Math.floor(sorted.length * 0.5)], + p95Ms: sorted[Math.floor(sorted.length * 0.95)], + } +} + +function sortAndLimitCounts( + counts: Map, + key: K, +): Array & {count: number}> { + return [...counts.entries()] + .map(([value, count]) => ({count, [key]: value}) as Record & {count: number}) + .sort((a, b) => b.count - a.count || a[key].localeCompare(b[key])) + .slice(0, TOP_LIMIT) +} + +function sortAndLimitGaps( + buckets: Map, +): QueryLogSummary['knowledgeGaps'] { + return [...buckets.entries()] + .map(([topic, {count, exampleQueries}]) => ({count, exampleQueries, topic})) + .sort((a, b) => b.count - a.count || a.topic.localeCompare(b.topic)) + .slice(0, TOP_LIMIT) +} + +function collectGapKeywords(query: string, buckets: Map): void { + const seen = new Set() + for (const token of query.toLowerCase().split(/[^a-z0-9]+/)) { + if (token.length < MIN_KEYWORD_LENGTH || STOPWORDS.has(token) || seen.has(token)) continue + seen.add(token) + + const bucket = buckets.get(token) ?? {count: 0, exampleQueries: []} + bucket.count += 1 + if (bucket.exampleQueries.length < MAX_EXAMPLE_QUERIES) { + bucket.exampleQueries.push(query) + } + + buckets.set(token, bucket) + } +} + +// ── Text formatting ───────────────────────────────────────────────────────── + +function formatSummaryText(summary: QueryLogSummary): string { + if (summary.totalQueries === 0) { + return EMPTY_TEXT_OUTPUT + } + + const cacheHits = CACHE_TIERS.reduce((sum, t) => sum + summary.byTier[`tier${t}`], 0) + const cachePct = Math.round(summary.cacheHitRate * 100) + const coveragePct = Math.round(summary.coverageRate * 100) + const matchedCount = summary.byStatus.completed - summary.queriesWithoutMatches + + const lines: string[] = [ + 'Query Recall Summary', + '====================', + `Total queries: ${summary.totalQueries}`, + ` Completed: ${summary.byStatus.completed}`, + ` Failed: ${summary.byStatus.error}`, + ` Cancelled: ${summary.byStatus.cancelled}`, + '', + `Cache hit rate: ${cachePct}% (${cacheHits}/${summary.byStatus.completed})`, + ] + + const maxTierLen = Math.max(...QUERY_LOG_TIERS.map((t) => `Tier ${t} (${QUERY_LOG_TIER_LABELS[t]}):`.length)) + for (const t of QUERY_LOG_TIERS) { + const label = `Tier ${t} (${QUERY_LOG_TIER_LABELS[t]}):` + lines.push(` ${label.padEnd(maxTierLen)} ${summary.byTier[`tier${t}`]}`) + } + + lines.push( + '', + 'Response time:', + ` Average: ${formatDuration(summary.responseTime.avgMs)}`, + ` p50: ${formatDuration(summary.responseTime.p50Ms)}`, + ` p95: ${formatDuration(summary.responseTime.p95Ms)}`, + '', + `Knowledge coverage: ${coveragePct}% (${matchedCount}/${summary.byStatus.completed} queries had relevant results)`, + ) + + if (summary.topTopics.length > 0) { + lines.push('', 'Top queried topics:') + for (const [i, t] of summary.topTopics.entries()) { + lines.push(` ${i + 1}. ${t.topic} — ${t.count} queries`) + } + } + + if (summary.topRecalledDocs.length > 0) { + lines.push('', 'Top recalled documents:') + for (const [i, d] of summary.topRecalledDocs.entries()) { + lines.push(` ${i + 1}. ${d.path} — ${d.count} queries`) + } + } + + if (summary.knowledgeGaps.length > 0) { + lines.push('', 'Knowledge gaps (asked but unanswered):') + for (const [i, g] of summary.knowledgeGaps.entries()) { + lines.push(` ${i + 1}. "${g.topic}" — ${g.count} unanswered queries`) + } + + lines.push(" → Run 'brv curate' on these topics to close the gap") + } + + return lines.join('\n') +} + +function formatDuration(ms: number): string { + if (ms >= 1000) { + return `${(ms / 1000).toFixed(1)}s` + } + + return `${Math.round(ms)}ms` } diff --git a/test/unit/infra/usecase/query-log-summary-narrative-formatter.test.ts b/test/unit/infra/usecase/query-log-summary-narrative-formatter.test.ts index 876e78289..6af21c8a2 100644 --- a/test/unit/infra/usecase/query-log-summary-narrative-formatter.test.ts +++ b/test/unit/infra/usecase/query-log-summary-narrative-formatter.test.ts @@ -30,8 +30,8 @@ function makeHappyPathSummary(overrides: Partial = {}): QueryLo return makeZeroSummary({ byStatus: {cancelled: 2, completed: 42, error: 3}, byTier: {tier0: 12, tier1: 6, tier2: 15, tier3: 10, tier4: 4, unknown: 0}, - cacheHitRate: 0.38, // 18/47 - coverageRate: 0.89, // 42/47 approx + cacheHitRate: 18 / 42, // 18/completed + coverageRate: 37 / 42, // (completed - queriesWithoutMatches) / completed knowledgeGaps: [ {count: 4, exampleQueries: ['how to deploy?', 'deployment steps'], topic: 'deployment pipeline'}, {count: 3, exampleQueries: ['rate limit'], topic: 'rate limiting'}, @@ -57,7 +57,7 @@ function makeZeroMatchedSummary(): QueryLogSummary { return makeHappyPathSummary({ coverageRate: 0, knowledgeGaps: [{count: 10, exampleQueries: ['foo'], topic: 'unknown topic'}], - queriesWithoutMatches: 47, + queriesWithoutMatches: 42, topRecalledDocs: [], totalMatchedDocs: 0, }) @@ -74,7 +74,7 @@ describe('formatQueryLogSummaryNarrative', () => { const out = formatQueryLogSummaryNarrative(summary) expect(out).to.equal( - 'No queries recorded in the last 24 hours. Your knowledge base is ready — try asking a question!', + 'No queries recorded in the selected period. Your knowledge base is ready — try asking a question!', ) }) }) @@ -90,16 +90,16 @@ describe('formatQueryLogSummaryNarrative', () => { expect(narrative).to.include('47 questions') }) - it('should report answered count as totalQueries minus queriesWithoutMatches', () => { - expect(narrative).to.include('answered 42 from curated knowledge') + it('should report answered count as completed minus queriesWithoutMatches', () => { + expect(narrative).to.include('answered 37 from curated knowledge') }) it('should include coverage rate as percentage', () => { - expect(narrative).to.include('89%') + expect(narrative).to.include('88%') }) it('should include cache hit rate as percentage', () => { - expect(narrative).to.include('38%') + expect(narrative).to.include('43%') }) it('should include average response time in seconds for multi-second times', () => { diff --git a/test/unit/infra/usecase/query-log-summary-use-case.test.ts b/test/unit/infra/usecase/query-log-summary-use-case.test.ts index 7d6888ef9..1613166e3 100644 --- a/test/unit/infra/usecase/query-log-summary-use-case.test.ts +++ b/test/unit/infra/usecase/query-log-summary-use-case.test.ts @@ -1,6 +1,7 @@ import {expect} from 'chai' import {createSandbox, type SinonSandbox, type SinonStub} from 'sinon' +import type {QueryLogEntry} from '../../../../src/server/core/domain/entities/query-log-entry.js' import type {IQueryLogStore} from '../../../../src/server/core/interfaces/storage/i-query-log-store.js' import {QueryLogSummaryUseCase} from '../../../../src/server/infra/usecase/query-log-summary-use-case.js' @@ -10,12 +11,30 @@ import {QueryLogSummaryUseCase} from '../../../../src/server/infra/usecase/query // ============================================================================ type MockTerminal = {log: SinonStub} +type MockStore = IQueryLogStore & { + getById: SinonStub + getNextId: SinonStub + list: SinonStub + save: SinonStub +} + +function makeStore(sandbox: SinonSandbox, entries: QueryLogEntry[] = []): MockStore { + return { + getById: sandbox.stub().resolves(null), + getNextId: sandbox.stub().resolves('qry-9999'), + list: sandbox.stub().resolves(entries), + save: sandbox.stub().resolves(), + } +} -function createUseCase(sandbox: SinonSandbox): {terminal: MockTerminal; useCase: QueryLogSummaryUseCase} { +function makeUseCase( + sandbox: SinonSandbox, + entries: QueryLogEntry[] = [], +): {store: MockStore; terminal: MockTerminal; useCase: QueryLogSummaryUseCase} { + const store = makeStore(sandbox, entries) const terminal: MockTerminal = {log: sandbox.stub()} - const queryLogStore = {} as IQueryLogStore - const useCase = new QueryLogSummaryUseCase({queryLogStore, terminal}) - return {terminal, useCase} + const useCase = new QueryLogSummaryUseCase({queryLogStore: store, terminal}) + return {store, terminal, useCase} } function loggedOutput(terminal: MockTerminal): string { @@ -25,70 +44,607 @@ function loggedOutput(terminal: MockTerminal): string { .join('\n') } +// ── Entity factories ──────────────────────────────────────────────────────── + +type CompletedEntry = Extract +type ErrorEntry = Extract +type CancelledEntry = Extract +type ProcessingEntry = Extract + +const T0 = 1_700_000_000_000 + +let idCounter = 0 +function nextId(): string { + idCounter += 1 + return `qry-${T0 + idCounter}` +} + +function makeCompleted(overrides: Partial = {}): CompletedEntry { + return { + completedAt: T0 + 100, + id: nextId(), + matchedDocs: [], + query: 'how is auth implemented?', + response: 'answer', + startedAt: T0, + status: 'completed', + taskId: 'task-1', + tier: 0, + timing: {durationMs: 100}, + ...overrides, + } +} + +function makeError(overrides: Partial = {}): ErrorEntry { + return { + completedAt: T0 + 50, + error: 'boom', + id: nextId(), + matchedDocs: [], + query: 'failing query', + startedAt: T0, + status: 'error', + taskId: 'task-1', + timing: {durationMs: 50}, + ...overrides, + } +} + +function makeCancelled(overrides: Partial = {}): CancelledEntry { + return { + completedAt: T0 + 10, + id: nextId(), + matchedDocs: [], + query: 'cancelled query', + startedAt: T0, + status: 'cancelled', + taskId: 'task-1', + ...overrides, + } +} + +function makeProcessing(overrides: Partial = {}): ProcessingEntry { + return { + id: nextId(), + matchedDocs: [], + query: 'in-flight query', + startedAt: T0, + status: 'processing', + taskId: 'task-1', + ...overrides, + } +} + // ============================================================================ // Tests // ============================================================================ -describe('QueryLogSummaryUseCase (stub)', () => { +describe('QueryLogSummaryUseCase', () => { let sandbox: SinonSandbox beforeEach(() => { sandbox = createSandbox() + idCounter = 0 }) afterEach(() => { sandbox.restore() }) - describe('format dispatch', () => { - it('defaults to text format and logs the empty placeholder', async () => { - const {terminal, useCase} = createUseCase(sandbox) + // ── 1. Empty entries ────────────────────────────────────────────────────── + describe('empty entries', () => { + it('returns a zero summary for json format with no entries', async () => { + const {terminal, useCase} = makeUseCase(sandbox, []) - await useCase.run({}) + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.totalQueries).to.equal(0) + expect(parsed.byStatus).to.deep.equal({cancelled: 0, completed: 0, error: 0}) + expect(parsed.byTier).to.deep.equal({tier0: 0, tier1: 0, tier2: 0, tier3: 0, tier4: 0, unknown: 0}) + expect(parsed.cacheHitRate).to.equal(0) + expect(parsed.coverageRate).to.equal(0) + expect(parsed.responseTime).to.deep.equal({avgMs: 0, p50Ms: 0, p95Ms: 0}) + expect(parsed.topTopics).to.deep.equal([]) + expect(parsed.topRecalledDocs).to.deep.equal([]) + expect(parsed.knowledgeGaps).to.deep.equal([]) + expect(parsed.totalMatchedDocs).to.equal(0) + expect(parsed.queriesWithoutMatches).to.equal(0) + }) + }) + + // ── 2-4, 17. Tier counts and cache hit rate ─────────────────────────────── + describe('byTier and cacheHitRate', () => { + it('single completed tier-0 entry yields cacheHitRate = 1.0', async () => { + const {terminal, useCase} = makeUseCase(sandbox, [makeCompleted({tier: 0})]) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.cacheHitRate).to.equal(1) + expect(parsed.byTier.tier0).to.equal(1) + }) + + it('counts byTier across mixed completed entries', async () => { + const entries = [ + makeCompleted({tier: 0}), + makeCompleted({tier: 1}), + makeCompleted({tier: 1}), + makeCompleted({tier: 2}), + makeCompleted({tier: 3}), + makeCompleted({tier: 4}), + makeCompleted({tier: undefined}), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.byTier).to.deep.equal({ + tier0: 1, + tier1: 2, + tier2: 1, + tier3: 1, + tier4: 1, + unknown: 1, + }) + }) + + it('cacheHitRate equals (tier0 + tier1) / totalCompleted', async () => { + const entries = [ + makeCompleted({tier: 0}), + makeCompleted({tier: 0}), + makeCompleted({tier: 1}), + makeCompleted({tier: 2}), + makeCompleted({tier: 3}), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.cacheHitRate).to.be.closeTo(0.6, 1e-9) // 3/5 + }) + + it('excludes processing entries from rate calculations and byStatus', async () => { + const entries = [makeCompleted({tier: 0}), makeCompleted({tier: 1}), makeProcessing(), makeProcessing()] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.totalQueries).to.equal(2) + expect(parsed.byStatus).to.deep.equal({cancelled: 0, completed: 2, error: 0}) + expect(parsed.cacheHitRate).to.equal(1) // 2/2 + }) + }) + + // ── 5-9. Response time percentiles ──────────────────────────────────────── + describe('responseTime percentiles', () => { + it('computes p50 with odd number of entries', async () => { + // 5 entries → floor(5*0.5) = index 2 + const entries = [100, 200, 300, 400, 500].map((ms) => makeCompleted({timing: {durationMs: ms}})) + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.responseTime.p50Ms).to.equal(300) + }) + + it('computes p50 with even number of entries', async () => { + // 4 entries → floor(4*0.5) = index 2 + const entries = [100, 200, 300, 400].map((ms) => makeCompleted({timing: {durationMs: ms}})) + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.responseTime.p50Ms).to.equal(300) + }) + + it('computes p95 with 20+ entries', async () => { + // 20 entries [100..2000] → floor(20*0.95) = index 19 → 2000 + const entries = Array.from({length: 20}, (_, i) => makeCompleted({timing: {durationMs: (i + 1) * 100}})) + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.responseTime.p95Ms).to.equal(2000) + }) + + it('with one entry, avg = p50 = p95 = that value', async () => { + const {terminal, useCase} = makeUseCase(sandbox, [makeCompleted({timing: {durationMs: 750}})]) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.responseTime).to.deep.equal({avgMs: 750, p50Ms: 750, p95Ms: 750}) + }) + + it('excludes error and cancelled entries from response time', async () => { + const entries = [ + makeCompleted({timing: {durationMs: 200}}), + makeError({timing: {durationMs: 9000}}), + makeCancelled(), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.responseTime.avgMs).to.equal(200) + expect(parsed.responseTime.p50Ms).to.equal(200) + }) + + it('excludes entries missing timing from calculation', async () => { + const entries = [ + makeCompleted({timing: {durationMs: 100}}), + makeCompleted({timing: undefined}), + makeCompleted({timing: {durationMs: 300}}), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.responseTime.avgMs).to.equal(200) // (100+300)/2 + }) + }) + + // ── 10-11. topTopics ────────────────────────────────────────────────────── + describe('topTopics', () => { + it('extracts first path segment and counts occurrences', async () => { + const entries = [ + makeCompleted({ + matchedDocs: [ + {path: 'authentication/oauth_flow.md', score: 0.9, title: 'oauth'}, + {path: 'authentication/token_storage.md', score: 0.8, title: 'tokens'}, + ], + }), + makeCompleted({ + matchedDocs: [{path: 'tool_system/registry.md', score: 0.7, title: 'registry'}], + }), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.topTopics).to.deep.equal([ + {count: 2, topic: 'authentication'}, + {count: 1, topic: 'tool_system'}, + ]) + }) + + it('sorts topTopics alphabetically when counts are equal', async () => { + const entries = [ + makeCompleted({matchedDocs: [{path: 'zebra/a.md', score: 1, title: 'z'}]}), + makeCompleted({matchedDocs: [{path: 'alpha/a.md', score: 1, title: 'a'}]}), + makeCompleted({matchedDocs: [{path: 'mango/a.md', score: 1, title: 'm'}]}), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) - expect(terminal.log.calledOnce).to.be.true - expect(terminal.log.firstCall.args[0]).to.equal('Query Recall Summary\n(no entries yet)') + const parsed = JSON.parse(loggedOutput(terminal)) + const topics = parsed.topTopics.map((t: {topic: string}) => t.topic) + expect(topics).to.deep.equal(['alpha', 'mango', 'zebra']) }) - it('explicit format: "text" logs the empty placeholder', async () => { - const {terminal, useCase} = createUseCase(sandbox) + it('sorts topTopics by count descending and limits to top 10', async () => { + const entries = Array.from({length: 12}, (_, i) => + makeCompleted({ + matchedDocs: [{path: `topic${i}/file.md`, score: 1, title: 'x'}], + }), + ) + // Boost topic5 so we can verify sort order + entries.push( + makeCompleted({ + matchedDocs: [ + {path: 'topic5/a.md', score: 1, title: 'a'}, + {path: 'topic5/b.md', score: 1, title: 'b'}, + {path: 'topic5/c.md', score: 1, title: 'c'}, + ], + }), + ) + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.topTopics).to.have.lengthOf(10) + expect(parsed.topTopics[0]).to.deep.equal({count: 4, topic: 'topic5'}) + }) + }) + + // ── 12-13. coverageRate ─────────────────────────────────────────────────── + describe('coverageRate', () => { + it('coverageRate = entriesWithMatchedDocs / totalCompleted', async () => { + const entries = [ + makeCompleted({matchedDocs: [{path: 'a/b.md', score: 1, title: 't'}]}), + makeCompleted({matchedDocs: [{path: 'a/c.md', score: 1, title: 't'}]}), + makeCompleted({matchedDocs: []}), + makeCompleted({matchedDocs: []}), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.coverageRate).to.equal(0.5) + expect(parsed.queriesWithoutMatches).to.equal(2) + }) + + it('coverageRate is 0 when no completed entry has matches', async () => { + const entries = [makeCompleted({matchedDocs: []}), makeCompleted({matchedDocs: []})] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.coverageRate).to.equal(0) + }) + }) + + // ── 14. Time range filtering forwarded to store ─────────────────────────── + describe('time range filtering', () => { + it('forwards after/before to store.list', async () => { + const {store, useCase} = makeUseCase(sandbox, []) + + await useCase.run({after: 1000, before: 2000, format: 'json'}) + + expect(store.list.calledOnce).to.be.true + expect(store.list.firstCall.args[0]).to.deep.include({after: 1000, before: 2000}) + }) + }) + + // ── 15. JSON output ─────────────────────────────────────────────────────── + describe('json output', () => { + it('includes all numeric metric fields', async () => { + const entries = [ + makeCompleted({ + matchedDocs: [{path: 'a/b.md', score: 1, title: 't'}], + tier: 0, + timing: {durationMs: 200}, + }), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed).to.have.all.keys([ + 'byStatus', + 'byTier', + 'cacheHitRate', + 'coverageRate', + 'knowledgeGaps', + 'period', + 'queriesWithoutMatches', + 'responseTime', + 'topRecalledDocs', + 'topTopics', + 'totalMatchedDocs', + 'totalQueries', + ]) + }) + }) + + // ── 16. Text output sections ────────────────────────────────────────────── + describe('text output', () => { + it('includes all sections for a populated summary', async () => { + const entries = [ + makeCompleted({ + matchedDocs: [{path: 'authentication/oauth.md', score: 1, title: 'oauth'}], + query: 'how does deployment pipeline work', + tier: 0, + timing: {durationMs: 1200}, + }), + makeCompleted({matchedDocs: [], query: 'rate limiting strategy'}), + makeError({}), + makeCancelled({}), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) await useCase.run({format: 'text'}) - expect(terminal.log.firstCall.args[0]).to.equal('Query Recall Summary\n(no entries yet)') + const out = loggedOutput(terminal) + expect(out).to.include('Query Recall Summary') + expect(out).to.include('Total queries:') + expect(out).to.include('Cache hit rate:') + expect(out).to.include('Tier 0 (exact cache hit)') + expect(out).to.include('Response time:') + expect(out).to.include('Knowledge coverage:') + expect(out).to.include('Top queried topics:') + expect(out).to.include('Top recalled documents:') + expect(out).to.include('Knowledge gaps') + expect(out).to.include("Run 'brv curate'") }) + }) - it('format: "narrative" logs the empty-state narrative from the formatter', async () => { - const {terminal, useCase} = createUseCase(sandbox) + // ── 18-20. topRecalledDocs ──────────────────────────────────────────────── + describe('topRecalledDocs', () => { + it('tracks full doc paths, not just first segment', async () => { + const entries = [ + makeCompleted({ + matchedDocs: [ + {path: 'authentication/oauth_flow.md', score: 1, title: 't'}, + {path: 'authentication/token_storage.md', score: 1, title: 't'}, + ], + }), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) - await useCase.run({format: 'narrative'}) + await useCase.run({format: 'json'}) - const output = loggedOutput(terminal) - expect(output).to.include('No queries recorded in the last 24 hours') - expect(output).to.include('knowledge base is ready') + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.topRecalledDocs).to.deep.equal([ + {count: 1, path: 'authentication/oauth_flow.md'}, + {count: 1, path: 'authentication/token_storage.md'}, + ]) }) - it('format: "json" logs valid JSON with a zero summary', async () => { - const {terminal, useCase} = createUseCase(sandbox) + it('sorts by count descending and limits to top 10', async () => { + const entries = Array.from({length: 12}, (_, i) => + makeCompleted({ + matchedDocs: [{path: `topic/doc${i}.md`, score: 1, title: 't'}], + }), + ) + // Boost doc5 + entries.push( + makeCompleted({matchedDocs: [{path: 'topic/doc5.md', score: 1, title: 't'}]}), + makeCompleted({matchedDocs: [{path: 'topic/doc5.md', score: 1, title: 't'}]}), + ) + const {terminal, useCase} = makeUseCase(sandbox, entries) await useCase.run({format: 'json'}) - const output = loggedOutput(terminal) - const parsed = JSON.parse(output) - expect(parsed.totalQueries).to.equal(0) - expect(parsed.byStatus).to.deep.equal({cancelled: 0, completed: 0, error: 0}) + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.topRecalledDocs).to.have.lengthOf(10) + expect(parsed.topRecalledDocs[0]).to.deep.equal({count: 3, path: 'topic/doc5.md'}) + }) + + it('deduplicates the same path across multiple entries', async () => { + const sharedPath = 'authentication/oauth_flow.md' + const entries = [ + makeCompleted({matchedDocs: [{path: sharedPath, score: 1, title: 't'}]}), + makeCompleted({matchedDocs: [{path: sharedPath, score: 1, title: 't'}]}), + makeCompleted({matchedDocs: [{path: sharedPath, score: 1, title: 't'}]}), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.topRecalledDocs).to.have.lengthOf(1) + expect(parsed.topRecalledDocs[0]).to.deep.equal({count: 3, path: sharedPath}) + }) + }) + + // ── 21-25. knowledgeGaps ────────────────────────────────────────────────── + describe('knowledgeGaps', () => { + it('filters completed entries with zero matched docs', async () => { + const entries = [ + makeCompleted({matchedDocs: [], query: 'deployment pipeline question'}), + makeCompleted({ + matchedDocs: [{path: 'a/b.md', score: 1, title: 't'}], + query: 'auth question', + }), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + const topics = parsed.knowledgeGaps.map((g: {topic: string}) => g.topic) + expect(topics).to.include('deployment') + expect(topics).to.not.include('auth') + }) + + it('extracts keywords and groups by frequency', async () => { + const entries = [ + makeCompleted({matchedDocs: [], query: 'deployment pipeline'}), + makeCompleted({matchedDocs: [], query: 'deployment scripts'}), + makeCompleted({matchedDocs: [], query: 'deployment workflow'}), + makeCompleted({matchedDocs: [], query: 'rate limiting'}), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + const top = parsed.knowledgeGaps[0] + expect(top.topic).to.equal('deployment') + expect(top.count).to.equal(3) + }) + + it('caps exampleQueries at 3 per topic', async () => { + const entries = [ + makeCompleted({matchedDocs: [], query: 'deployment one'}), + makeCompleted({matchedDocs: [], query: 'deployment two'}), + makeCompleted({matchedDocs: [], query: 'deployment three'}), + makeCompleted({matchedDocs: [], query: 'deployment four'}), + makeCompleted({matchedDocs: [], query: 'deployment five'}), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + const top = parsed.knowledgeGaps.find((g: {topic: string}) => g.topic === 'deployment') + expect(top.exampleQueries).to.have.lengthOf(3) + }) + + it('excludes error and cancelled entries from gaps', async () => { + const entries = [ + makeError({matchedDocs: [], query: 'errored deployment'}), + makeCancelled({matchedDocs: [], query: 'cancelled deployment'}), + makeCompleted({ + matchedDocs: [{path: 'a/b.md', score: 1, title: 't'}], + query: 'covered question', + }), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) + expect(parsed.knowledgeGaps).to.deep.equal([]) + }) + + it('returns empty array when there are no gaps', async () => { + const entries = [ + makeCompleted({matchedDocs: [{path: 'a/b.md', score: 1, title: 't'}]}), + makeCompleted({matchedDocs: [{path: 'a/c.md', score: 1, title: 't'}]}), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'json'}) + + const parsed = JSON.parse(loggedOutput(terminal)) expect(parsed.knowledgeGaps).to.deep.equal([]) - expect(parsed.queriesWithoutMatches).to.equal(0) }) }) - describe('options forwarding', () => { - it('still produces output when after/before are provided', async () => { - const {terminal, useCase} = createUseCase(sandbox) + // ── 26-28. Format dispatch ──────────────────────────────────────────────── + describe('format dispatch', () => { + it('format: "narrative" dispatches to formatQueryLogSummaryNarrative with computed summary', async () => { + const entries = [ + makeCompleted({ + matchedDocs: [{path: 'authentication/oauth.md', score: 1, title: 'oauth'}], + query: 'how does auth work', + tier: 0, + timing: {durationMs: 500}, + }), + ] + const {terminal, useCase} = makeUseCase(sandbox, entries) + + await useCase.run({format: 'narrative'}) + + const out = loggedOutput(terminal) + expect(out).to.include('1 question') + expect(out).to.include('answered 1 from curated knowledge') + }) + + it('format: "narrative" with empty entries logs the empty-state message', async () => { + const {terminal, useCase} = makeUseCase(sandbox, []) + + await useCase.run({format: 'narrative'}) + + const out = loggedOutput(terminal) + expect(out).to.include('No queries recorded') + expect(out).to.include('knowledge base is ready') + }) - await useCase.run({after: 1000, before: 2000, format: 'text'}) + it('defaults to text format when format is undefined', async () => { + const {terminal, useCase} = makeUseCase(sandbox, []) + + await useCase.run({}) - expect(terminal.log.calledOnce).to.be.true + const out = loggedOutput(terminal) + expect(out).to.equal('Query Recall Summary\n(no entries yet)') }) }) })