diff --git a/CHANGELOG.md b/CHANGELOG.md index 149b70e8..56fb7ceb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ Cross-package release notes for relayburn. Package changelogs contain package-le ## [Unreleased] +- **BREAKING (`relayburn-sdk`):** the published Rust SDK no longer re-exports its low-level `analyze`-layer internals (detector/aggregator functions and helper types such as `PricingTable`, `CompareTable`, `CompareCell`) — these were never the intended embedding surface. Embed through the verb layer instead: `LedgerHandle` methods / `summary_report` / `hotspots` / `compare`. CLI, MCP, and `@relayburn/sdk` behavior is unchanged. +- `burn compare` cost figures now use canonical decimal rounding (`{:.N}`/`toFixed` semantics) instead of float-multiply rounding, so cells/totals/buckets can shift by one in the last reported digit at exact ties; affects the `compare` verb's JSON for all consumers (CLI, MCP, `@relayburn/sdk`). +- Fidelity summaries (`fidelity` block in `summary`/`compare` JSON) now emit `byClass` / `byGranularity` / `missingCoverage` keys in a stable order instead of a randomized per-run order, so output is reproducible across runs (diff-, cache-, and snapshot-friendly). - `burn` subagent-tree views now require a re-ingest to render pre-Root-emission event logs (legacy reconstruction path removed). ## [3.4.0] - 2026-06-20 diff --git a/crates/relayburn-cli/src/commands/compare.rs b/crates/relayburn-cli/src/commands/compare.rs index c8d04075..5a344864 100644 --- a/crates/relayburn-cli/src/commands/compare.rs +++ b/crates/relayburn-cli/src/commands/compare.rs @@ -1,8 +1,9 @@ //! `burn compare ` — per-(model, activity) cost -//! comparison table. Thin presenter over the -//! `relayburn_sdk::analyze::compare` building blocks (`build_compare_table` -//! plus `compare_from_archive`); the heavy lifting lives in the SDK so the -//! MCP server can reuse it. +//! comparison table. Thin presenter over the `LedgerHandle::compare` / +//! `compare_timeseries` SDK verbs; the full pipeline (query → provider filter +//! → fidelity gate → pricing → per-cell aggregation) lives in the SDK so the +//! MCP server can reuse it. This file only adapts the verb's `CompareResult` +//! into the JSON / CSV / TTY wire shapes. //! //! TS source of truth: `packages/cli/src/commands/compare.ts`. The wire //! shape (cells ordering, rounding rules, fidelity-summary key order) @@ -21,21 +22,21 @@ //! `burn ingest && burn compare`; the steady-state setup is to run //! `burn ingest --watch` once per host so every read verb sees current data. //! -//! Filter wiring: `--project` / `--session` / `--since` lower into the -//! `Query` struct directly; `--workflow` / `--agent` fold through -//! `Query.enrichment` (`workflowId` / `agentId` keys, both required when -//! both flags are passed); `--provider` is a post-query CSV allow-list -//! resolved via `provider_for`, matching `summary --by-provider`'s -//! synthetic-rule-aware classification. +//! Filter wiring: `--project` / `--session` / `--since` / `--workflow` / +//! `--agent` / `--provider` all flow into the verb's `CompareOptions`. The +//! verb lowers `--workflow` / `--agent` through stamp enrichment +//! (`workflowId` / `agentId` keys, both required when both flags are passed) +//! and applies `--provider` as a post-query CSV allow-list resolved via +//! `provider_for`, matching `summary --by-provider`'s synthetic-rule-aware +//! classification. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeSet, HashMap}; use anyhow::{anyhow, Result}; use relayburn_sdk::{ - build_compare_table, has_minimum_fidelity, load_pricing, normalize_since, provider_for, - summarize_fidelity, AnalyzeCompareOptions as CompareOptions, CompareCell, CompareTable, - EnrichedTurn, FidelityClass, FidelitySummary, Ledger, LedgerOpenOptions, ProviderFilter, Query, - UsageGranularity, DEFAULT_MIN_SAMPLE, + normalize_since, CompareCellResult, CompareExcludedBreakdown, CompareOptions, CompareResult, + FidelityClass, FidelitySummary, Ledger, LedgerOpenOptions, UsageGranularity, + DEFAULT_MIN_SAMPLE, }; use serde_json::{json, Value}; @@ -53,14 +54,6 @@ const FIDELITY_CHOICES: &[&str] = &[ "partial", ]; -const FIDELITY_ORDER: &[&str] = &[ - "cost-only", - "aggregate-only", - "partial", - "usage-only", - "full", -]; - const NEEDS_MODELS_MSG: &str = "compare: needs at least 2 models. Run `burn summary --by-provider` (or `burn summary --by-tool`) to see which models have data."; @@ -148,8 +141,10 @@ fn run_inner(globals: &GlobalArgs, args: CompareArgs) -> Result { } // 4. Provider filter. Lower-cased CSV; turns whose effective provider - // (per `provider_for`) isn't in the set get dropped after the ledger - // query but before the fidelity gate, matching the TS pipeline order. + // (resolved inside the verb via `provider_for`) get dropped after the + // ledger query but before the fidelity gate, matching the TS pipeline + // order. Parsed here so a malformed `--provider` routes through the + // same error envelope as the other argument validations. let provider_filter = parse_provider_filter(args.provider.as_deref())?; // 5. min-sample. @@ -162,33 +157,13 @@ fn run_inner(globals: &GlobalArgs, args: CompareArgs) -> Result { // the Rust SDK is SQLite-native and has no archive layer to bypass. let _ = args.no_archive; - // 7. Build the Query. - let mut q = Query::default(); - if let Some(s) = normalize_since(args.since.as_deref())? { - q.since = Some(s); - } - if let Some(p) = args.project.as_deref() { - q.project = Some(p.to_string()); - } - if let Some(s) = args.session.as_deref() { - q.session_id = Some(s.to_string()); - } - // `workflow` / `agent` fold through stamp enrichment. Both keys live in - // the same `Enrichment` map (`workflowId`, `agentId`), and the ledger's - // `Query.enrichment` predicate requires every key/value pair to match — - // so passing both narrows to the intersection. - let mut enrichment = BTreeMap::new(); - if let Some(workflow) = args.workflow.as_deref() { - enrichment.insert("workflowId".to_string(), workflow.to_string()); - } - if let Some(agent) = args.agent.as_deref() { - enrichment.insert("agentId".to_string(), agent.to_string()); - } - if !enrichment.is_empty() { - q.enrichment = Some(enrichment); - } + // 7. Validate `--since` up front so a malformed duration routes through + // the same error envelope as the other argument validations. The verb + // re-parses it internally from `opts.since`; this is purely the + // early-fail gate (the parsed value is discarded). + let _ = normalize_since(args.since.as_deref())?; - // 8. Open ledger and walk turns. + // 8. Open ledger. let progress = TaskProgress::new(globals, "compare"); let ledger_opts = match globals.ledger_path.as_deref() { Some(p) => LedgerOpenOptions::with_home(p), @@ -200,20 +175,17 @@ fn run_inner(globals: &GlobalArgs, args: CompareArgs) -> Result { // `--bucket` switches to a per-bucket time-series via the SDK verb. // Parsing/validation already happened above, before the ledger was opened. if let Some(bucket_secs) = bucket_secs { - let provider = provider_filter - .as_ref() - .map(|f| f.iter().cloned().collect::>()); progress.set_task("building comparison time-series"); let series = handle .compare_timeseries( - relayburn_sdk::CompareOptions { + CompareOptions { models: models.clone(), session: args.session.clone(), project: args.project.clone(), since: args.since.clone(), workflow: args.workflow.clone(), agent: args.agent.clone(), - provider, + provider: provider_filter.clone(), min_sample: Some(min_sample), min_fidelity: Some(min_fidelity), ledger_home: None, @@ -241,65 +213,40 @@ fn run_inner(globals: &GlobalArgs, args: CompareArgs) -> Result { return Ok(0); } - progress.set_task("loading turns"); - let queried_turns: Vec = handle.raw().query_turns(&q)?; - - // 9. Drop turns whose effective provider isn't in the allow-list. The - // provider is resolved via `provider_for` (synthetic-rule first, - // then `provider/model` prefix, then collector-implied) so the CLI - // matches `summary --by-provider` semantics 1:1. - let filtered_by_provider: Vec = match provider_filter.as_ref() { - Some(filter) => queried_turns - .into_iter() - .filter(|et| { - let provider = provider_for(&et.turn).provider; - filter.contains(&provider.to_ascii_lowercase()) - }) - .collect(), - None => queried_turns, - }; - - // 10. Fidelity summary is computed BEFORE the fidelity gate so the - // `summary` block in the JSON envelope reflects the queried slice. - let fidelity_summary = summarize_fidelity( - &filtered_by_provider - .iter() - .map(|et| et.turn.clone()) - .collect::>(), - ); - let filtered_turns: Vec = if matches!(min_fidelity, FidelityClass::Partial) { - filtered_by_provider - } else { - filtered_by_provider - .into_iter() - .filter(|et| has_minimum_fidelity(et.turn.fidelity.as_ref(), min_fidelity)) - .collect() - }; - let analyzed_turns = filtered_turns.len(); - - // 11. Build the compare table. - let pricing = load_pricing(None); - let opts = CompareOptions { - pricing: &pricing, - models: Some(models.clone()), - min_sample: Some(min_sample), - }; + // 9. Run the comparison through the SDK verb. The verb owns the full + // pipeline — query → provider filter (`provider_for`) → fidelity + // summary (over the post-provider, pre-gate slice) → fidelity gate → + // pricing → per-cell aggregation — so the CLI is a pure presenter over + // the returned `CompareResult`. progress.set_task("building comparison"); - let table = build_compare_table(&filtered_turns, &opts); + let result = handle + .compare(CompareOptions { + models: models.clone(), + session: args.session.clone(), + project: args.project.clone(), + since: args.since.clone(), + workflow: args.workflow.clone(), + agent: args.agent.clone(), + provider: provider_filter, + min_sample: Some(min_sample), + min_fidelity: Some(min_fidelity), + ledger_home: None, + }) + .inspect_err(|_| progress.finish_and_clear())?; progress.finish_and_clear(); - // 12. Render. + // 10. Render. if globals.json { - let v = build_json(&table, analyzed_turns, min_fidelity, &fidelity_summary); + let v = build_json(&result); render_json(&v)?; return Ok(0); } if args.csv { - let csv = render_csv(&table); + let csv = render_csv(&result); print!("{csv}"); return Ok(0); } - let tty = render_tty(&table, analyzed_turns, min_fidelity, &fidelity_summary); + let tty = render_tty(&result); print!("{tty}"); Ok(0) } @@ -308,16 +255,19 @@ fn run_inner(globals: &GlobalArgs, args: CompareArgs) -> Result { // helpers // --------------------------------------------------------------------------- -/// Parse `--provider` CSV → lower-cased `ProviderFilter`. Mirrors the -/// `summary --provider` parser: trim entries, drop empties, lower-case for -/// case-insensitive matches, and reject an all-empty list with the same -/// error shape (`burn: --provider requires a value`). Returns `Ok(None)` -/// when the flag wasn't passed. -fn parse_provider_filter(raw: Option<&str>) -> Result> { +/// Parse `--provider` CSV → lower-cased allow-list passed to the verb's +/// `CompareOptions.provider`. Mirrors the `summary --provider` parser: trim +/// entries, drop empties, lower-case for case-insensitive matches, and reject +/// an all-empty list with the same error shape +/// (`burn: --provider requires a value`). Returns `Ok(None)` when the flag +/// wasn't passed. The verb re-normalizes (trim / lowercase / dedupe) +/// internally, so this only needs to reject the all-empty case here to keep +/// the argument-validation error envelope. +fn parse_provider_filter(raw: Option<&str>) -> Result>> { let Some(raw) = raw else { return Ok(None); }; - let providers: ProviderFilter = raw + let providers: Vec = raw .split(',') .map(|s| s.trim().to_ascii_lowercase()) .filter(|s| !s.is_empty()) @@ -422,70 +372,60 @@ fn round_opt(n: Option, digits: usize) -> Value { } // --------------------------------------------------------------------------- -// CompareExcludedBreakdown +// cell lookup helpers // --------------------------------------------------------------------------- -#[derive(Default)] -struct ExcludedBreakdown { - total: u64, - aggregate_only: u64, - cost_only: u64, - partial: u64, - usage_only: u64, +/// Empty-cell stand-in for `(model, category)` pairs the verb didn't emit. +/// `build_compare_table` seeds every pair, so in practice this is only a +/// defensive fallback (mirrors the analyze-layer `empty_cell`). +fn empty_cell(model: &str, category: &str) -> CompareCellResult { + CompareCellResult { + model: model.to_string(), + category: category.to_string(), + turns: 0, + edit_turns: 0, + one_shot_turns: 0, + priced_turns: 0, + total_cost: 0.0, + cost_per_turn: None, + one_shot_rate: None, + cache_hit_rate: None, + median_retries: None, + no_data: true, + insufficient_sample: false, + } } -fn compute_excluded(summary: &FidelitySummary, minimum: FidelityClass) -> ExcludedBreakdown { - let mut out = ExcludedBreakdown::default(); - if matches!(minimum, FidelityClass::Partial) { - return out; - } - let need = FIDELITY_ORDER +/// Index the verb's flat `cells` vec by `(model, category)` so the renderers +/// can look up the same per-cell data the nested analyze-layer `CompareTable` +/// used to provide. +fn index_cells(result: &CompareResult) -> HashMap<(&str, &str), &CompareCellResult> { + result + .cells .iter() - .position(|c| *c == minimum.wire_str()) - .unwrap_or(0); - for (i, key) in FIDELITY_ORDER.iter().enumerate() { - if i >= need { - continue; - } - let cls = parse_fidelity(key).unwrap(); - let n = summary.by_class.get(&cls).copied().unwrap_or(0); - if n == 0 { - continue; - } - out.total += n; - match *key { - "aggregate-only" => out.aggregate_only += n, - "cost-only" => out.cost_only += n, - "partial" => out.partial += n, - "usage-only" => out.usage_only += n, - _ => {} - } - } - out + .map(|c| ((c.model.as_str(), c.category.as_str()), c)) + .collect() } // --------------------------------------------------------------------------- // JSON envelope // --------------------------------------------------------------------------- -fn build_json( - table: &CompareTable, - analyzed_turns: usize, - minimum: FidelityClass, - summary: &FidelitySummary, -) -> Value { - let excluded = compute_excluded(summary, minimum); +fn build_json(result: &CompareResult) -> Value { + let index = index_cells(result); // Cells in (model × category) iteration order; matches the TS // `for m of models / for cat of categories` walk. - let mut cells: Vec = Vec::with_capacity(table.models.len() * table.categories.len()); - for m in &table.models { - for cat in &table.categories { - let c = table - .cells - .get(m) - .and_then(|by_cat| by_cat.get(cat)) - .cloned() - .unwrap_or_else(empty_cell); + let mut cells: Vec = Vec::with_capacity(result.models.len() * result.categories.len()); + for m in &result.models { + for cat in &result.categories { + let owned; + let c = match index.get(&(m.as_str(), cat.as_str())) { + Some(c) => *c, + None => { + owned = empty_cell(m, cat); + &owned + } + }; cells.push(json!({ "model": m, "category": cat, @@ -508,26 +448,30 @@ fn build_json( // preserves insertion order). Build with a serde_json::Map so the // `preserve_order` feature on serde_json keeps insertion order. let mut totals = serde_json::Map::new(); - for m in &table.models { - let totals_for = table.totals.get(m).cloned().unwrap_or_default(); + for m in &result.models { + let totals_for = result.totals.get(m); + let (turns, total_cost) = totals_for + .map(|t| (t.turns, t.total_cost)) + .unwrap_or((0, 0.0)); totals.insert( m.clone(), json!({ - "turns": totals_for.turns, - "totalCost": f64_to_json(totals_for.total_cost), + "turns": turns, + "totalCost": f64_to_json(total_cost), }), ); } + let excluded = &result.fidelity.excluded; json!({ - "analyzedTurns": analyzed_turns, - "minSample": table.min_sample, - "models": &table.models, - "categories": &table.categories, + "analyzedTurns": result.analyzed_turns, + "minSample": result.min_sample, + "models": &result.models, + "categories": &result.categories, "totals": Value::Object(totals), "cells": cells, "fidelity": { - "minimum": minimum.wire_str(), + "minimum": result.fidelity.minimum.wire_str(), "excluded": { "total": excluded.total, "aggregateOnly": excluded.aggregate_only, @@ -535,7 +479,7 @@ fn build_json( "partial": excluded.partial, "usageOnly": excluded.usage_only, }, - "summary": fidelity_summary_to_value(summary), + "summary": fidelity_summary_to_value(&result.fidelity.summary), } }) } @@ -602,27 +546,12 @@ fn fidelity_summary_to_value(s: &FidelitySummary) -> Value { Value::Object(out) } -fn empty_cell() -> CompareCell { - CompareCell { - turns: 0, - edit_turns: 0, - one_shot_turns: 0, - priced_turns: 0, - total_cost: 0.0, - cost_per_turn: None, - one_shot_rate: None, - cache_hit_rate: None, - median_retries: None, - no_data: true, - insufficient_sample: false, - } -} - // --------------------------------------------------------------------------- // CSV // --------------------------------------------------------------------------- -fn render_csv(table: &CompareTable) -> String { +fn render_csv(result: &CompareResult) -> String { + let index = index_cells(result); let header = [ "model", "category", @@ -640,14 +569,16 @@ fn render_csv(table: &CompareTable) -> String { ]; let mut rows: Vec = Vec::new(); rows.push(header.join(",")); - for m in &table.models { - for cat in &table.categories { - let c = table - .cells - .get(m) - .and_then(|by_cat| by_cat.get(cat)) - .cloned() - .unwrap_or_else(empty_cell); + for m in &result.models { + for cat in &result.categories { + let owned; + let c = match index.get(&(m.as_str(), cat.as_str())) { + Some(c) => *c, + None => { + owned = empty_cell(m, cat); + &owned + } + }; let row = vec![ csv_cell(m), csv_cell(cat), @@ -699,7 +630,7 @@ fn num_csv(n: f64, digits: usize) -> String { // TTY // --------------------------------------------------------------------------- -fn cell_fields(c: &CompareCell) -> [String; 3] { +fn cell_fields(c: &CompareCellResult) -> [String; 3] { if c.no_data { return [DASH.to_string(), DASH.to_string(), DASH.to_string()]; } @@ -715,52 +646,44 @@ fn cell_fields(c: &CompareCell) -> [String; 3] { [turns, cost, one_shot] } -fn render_tty( - table: &CompareTable, - analyzed_turns: usize, - minimum: FidelityClass, - summary: &FidelitySummary, -) -> String { +fn render_tty(result: &CompareResult) -> String { + let minimum = result.fidelity.minimum; + let index = index_cells(result); + // `(model, category)` lookup with an empty-cell fallback for pairs the + // verb didn't emit (it always emits the full grid, so this is defensive). + let cell_for = |m: &str, cat: &str| -> CompareCellResult { + index + .get(&(m, cat)) + .map(|c| (*c).clone()) + .unwrap_or_else(|| empty_cell(m, cat)) + }; + let mut lines: Vec = Vec::new(); lines.push(String::new()); lines.push(format!( "turns analyzed: {}", - format_uint(analyzed_turns as u64) + format_uint(result.analyzed_turns) )); - let excluded = compute_excluded(summary, minimum); + let excluded = &result.fidelity.excluded; if excluded.total > 0 { - lines.push(format_excluded_note(&excluded, minimum)); + lines.push(format_excluded_note(excluded, minimum)); } lines.push(String::new()); - if table.models.is_empty() || table.categories.is_empty() { + if result.models.is_empty() || result.categories.is_empty() { lines .push("no data to compare (need turns spanning ≥1 model and ≥1 activity).".to_string()); lines.push(String::new()); return lines.join("\n"); } - let sub_header = build_sub_header(&table.models); - - let owned_empty = empty_cell(); - let cell_for = |m: &str, cat: &str| -> CompareCell { - table - .cells - .get(m) - .and_then(|by| by.get(cat)) - .cloned() - .unwrap_or_else(empty_cell) - }; - // Suppress the unused-variable warning on `owned_empty`; it's only - // referenced when we run a corner case where neither cells.get nor - // by_cat.get is hit, which the table builder doesn't produce today. - let _ = &owned_empty; + let sub_header = build_sub_header(&result.models); let mut data_rows: Vec> = Vec::new(); - for cat in &table.categories { + for cat in &result.categories { let mut row: Vec = vec![cat.clone()]; - for m in &table.models { + for m in &result.models { let cell = cell_for(m, cat); let [a, b, c] = cell_fields(&cell); row.push(a); @@ -781,11 +704,11 @@ fn render_tty( // Widen the last column of each model's group to fit the (possibly // longer) display name. Mirrors the TS path's group-line padding. - for mi in 0..table.models.len() { + for mi in 0..result.models.len() { let start = 1 + mi * 3; let group_width = widths[start] + SEP.len() + widths[start + 1] + SEP.len() + widths[start + 2]; - let name = display_model_name(&table.models[mi]); + let name = display_model_name(&result.models[mi]); let name_w = display_width(name); if name_w > group_width { widths[start + 2] += name_w - group_width; @@ -794,11 +717,11 @@ fn render_tty( // Group-name line. let mut group_line: Vec = vec![pad_end("", widths[0])]; - for mi in 0..table.models.len() { + for mi in 0..result.models.len() { let start = 1 + mi * 3; let group_width = widths[start] + SEP.len() + widths[start + 1] + SEP.len() + widths[start + 2]; - let name = display_model_name(&table.models[mi]); + let name = display_model_name(&result.models[mi]); group_line.push(pad_end(name, group_width)); } lines.push(rstrip(&group_line.join(SEP))); @@ -813,12 +736,12 @@ fn render_tty( // Coverage notes. let mut notes: Vec = Vec::new(); - for cat in &table.categories { - let any_has_data = table.models.iter().any(|m| !cell_for(m, cat).no_data); + for cat in &result.categories { + let any_has_data = result.models.iter().any(|m| !cell_for(m, cat).no_data); if !any_has_data { continue; } - for m in &table.models { + for m in &result.models { let cell = cell_for(m, cat); if cell.no_data { notes.push(format!( @@ -830,7 +753,7 @@ fn render_tty( "low {} sample in '{cat}' ({} turns < {}) — treat as indicative.", display_model_name(m), cell.turns, - table.min_sample + result.min_sample )); } } @@ -851,17 +774,21 @@ fn render_tty( // Per-model totals. lines.push(String::new()); - for m in &table.models { - let tot = table.totals.get(m).cloned().unwrap_or_default(); - let total_cost = if tot.turns > 0 { - format_usd(tot.total_cost) + for m in &result.models { + let (turns, total_cost_raw) = result + .totals + .get(m) + .map(|t| (t.turns, t.total_cost)) + .unwrap_or((0, 0.0)); + let total_cost = if turns > 0 { + format_usd(total_cost_raw) } else { DASH.to_string() }; lines.push(format!( "{}: {} turns, {} total", display_model_name(m), - format_uint(tot.turns), + format_uint(turns), total_cost )); } @@ -916,7 +843,7 @@ fn display_model_name(m: &str) -> &str { } } -fn format_excluded_note(excluded: &ExcludedBreakdown, minimum: FidelityClass) -> String { +fn format_excluded_note(excluded: &CompareExcludedBreakdown, minimum: FidelityClass) -> String { let mut parts: Vec = Vec::new(); if excluded.aggregate_only > 0 { parts.push(format!("{} aggregate-only", excluded.aggregate_only)); @@ -969,13 +896,14 @@ mod tests { } #[test] - fn parse_provider_filter_trims_lowercases_and_dedupes() { + fn parse_provider_filter_trims_lowercases_and_drops_empties() { + // The CLI parser trims / lowercases / drops empties; deduping is left + // to the verb's `normalize_provider_filter`, so the raw entries + // (including the repeat) flow through as a `Vec`. let got = parse_provider_filter(Some(" Anthropic,OPENAI ,, anthropic")) .unwrap() .unwrap(); - assert!(got.contains("anthropic")); - assert!(got.contains("openai")); - assert_eq!(got.len(), 2, "duplicates should collapse: got {got:?}"); + assert_eq!(got, vec!["anthropic", "openai", "anthropic"]); } #[test] diff --git a/crates/relayburn-cli/src/commands/hotspots.rs b/crates/relayburn-cli/src/commands/hotspots/human.rs similarity index 52% rename from crates/relayburn-cli/src/commands/hotspots.rs rename to crates/relayburn-cli/src/commands/hotspots/human.rs index 53972439..5c59b647 100644 --- a/crates/relayburn-cli/src/commands/hotspots.rs +++ b/crates/relayburn-cli/src/commands/hotspots/human.rs @@ -1,570 +1,21 @@ -//! `burn hotspots` — surface high-cost / high-overhead hotspots from -//! the ledger. -//! -//! Thin presenter over [`relayburn_sdk::hotspots`]. Mirrors -//! `packages/cli/src/commands/hotspots.ts` for the default attribution -//! flow that drives the golden snapshots; the broader TS surface -//! (`--patterns`, `--findings`, `--session` per-session view, -//! `--provider`, `--workflow`) is enumerated as flag wiring + a -//! stub-mode error path. -//! -//! ## Wiring -//! -//! 1. Open a [`relayburn_sdk::LedgerHandle`] honoring `--ledger-path` / -//! `RELAYBURN_HOME`. -//! 2. Run [`relayburn_sdk::ingest_all`] with TTY-only progress. -//! 3. Call [`relayburn_sdk::hotspots`] (verb-form) with the resolved -//! [`relayburn_sdk::HotspotsOptions`]. The SDK enforces the coverage -//! gate, picks the `Sized` vs `EvenSplit` attribution method per -//! session, and emits the discriminated union; for the default flow -//! we expect [`relayburn_sdk::HotspotsResult::Attribution`] and -//! unwrap that branch. -//! 4. Render JSON or human format. JSON output drops the `kind` -//! discriminator and emits the inner `HotspotsAttributionResult` -//! shape directly (TS contract). +//! Human-readable table rendering for `burn hotspots`. -use clap::{Args, ValueEnum}; use relayburn_sdk::{ - hotspots as sdk_hotspots, ingest_all, AttributionMethod, BashAggregation, BashVerbAggregation, - FileAggregation, HotspotsAttributionResult, HotspotsExcludedBreakdown, - HotspotsExcludedSourceRow, HotspotsGroupBy, HotspotsOptions, HotspotsResult, - HotspotsSessionTotal, Ledger, LedgerOpenOptions, McpServerAggregation, SubagentAggregation, - WasteFinding, WasteSeverity, + AttributionMethod, BashAggregation, BashVerbAggregation, FileAggregation, + HotspotsAttributionResult, HotspotsExcludedBreakdown, HotspotsExcludedSourceRow, + HotspotsResult, McpServerAggregation, SubagentAggregation, WasteFinding, WasteSeverity, }; -use serde_json::{json, Map, Value}; -use crate::cli::GlobalArgs; -use crate::render::error::report_error; -use crate::render::format::{coerce_whole_f64_to_int, format_uint, format_usd, render_table}; -use crate::render::json::render_json; -use crate::render::progress::TaskProgress; +use crate::render::format::{format_uint, format_usd, render_table}; -const DEFAULT_TOP_N: usize = 10; +use super::*; -/// Per-command flags for `burn hotspots`. Mirrors the TS surface in -/// `packages/cli/src/commands/hotspots.ts`. -#[derive(Debug, Clone, Args)] -pub struct HotspotsArgs { - /// Slice the ledger to events at or after ``. ISO timestamp - /// or relative range (`24h`, `7d`, `4w`, `2m`). - #[arg(long, value_name = "WHEN")] - pub since: Option, - - /// Restrict to a single project. - #[arg(long, value_name = "PROJECT")] - pub project: Option, - - /// Restrict to a single session id (or pass without a value to drop - /// into the per-session attribution view). - #[arg(long, value_name = "SESSION_ID", num_args = 0..=1, default_missing_value = "")] - pub session: Option, - - /// Filter by enrichment workflow id. - #[arg(long, value_name = "WORKFLOW_ID")] - pub workflow: Option, - - /// Provider filter (CSV of provider names; case-insensitive). - #[arg(long, value_name = "PROVIDERS")] - pub provider: Option, - - /// Show all rows in human mode instead of capping at the default - /// top-N (10). - #[arg(long)] - pub all: bool, - - /// Group by a single dimension. Defaults to the full attribution - /// view; pass `bash`, `bash-verb`, `file`, or `subagent` to focus - /// a single rollup. - #[arg(long = "group-by", value_name = "DIM")] - pub group_by: Option, - - /// Comma-separated waste-pattern detectors to run instead of the - /// attribution view. Pass without a value to enable every detector. - #[arg(long, value_name = "PATTERNS", num_args = 0..=1, default_missing_value = "")] - pub patterns: Option, - - /// Render the unified `findings` view rather than the per-detector - /// summary. Implies `--patterns` if it isn't already set. - #[arg(long)] - pub findings: bool, - - /// Surface session relationship drift on top of the default attribution - /// view. Currently a stub in the Rust port — the relationship drift - /// query verb is not yet exposed by the SDK. - #[arg(long = "explain-drift")] - pub explain_drift: bool, - - /// Ranking dimension for the per-tool tables (files, bash, bash verbs, - /// subagents). `cost` (default) keeps the historical USD-descending - /// order; `bytes` sorts by `totalOutputBytes` so blowouts that get - /// truncated to ~0 tokens still surface (#436). JSON output is - /// unaffected — both rankings ship every field; downstream consumers - /// pick their own sort. - #[arg(long = "rank-by", value_name = "DIM", value_enum, default_value_t = RankBy::Cost)] - pub rank_by: RankBy, - - /// Run a pre-query ingest sweep so hotspots reflects freshly appended - /// sessions. Off by default: `hotspots` is a read verb, and a full-store - /// sweep re-stats every session file under every harness store — seconds - /// on a large ledger. Keep the ledger current out of band with - /// `burn ingest --watch` (or the Claude Stop hook); pass `--ingest` only - /// for a one-off freshen. Mirrors `burn summary --ingest`. - #[arg(long = "ingest")] - pub ingest: bool, -} - -/// Sort dimension for the per-tool human-mode tables. Mirrors `--rank-by`. -/// `ValueEnum` so clap validates at parse time — invalid values fail before -/// any ingest work runs, in both human and `--json` modes. -#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] -pub enum RankBy { - Cost, - Bytes, -} - -// Detector kinds the SDK's `run_hotspots_findings` filter expects. These are -// the *finding* kind strings emitted by `WasteFinding.kind` (e.g. -// `compaction-loss`, `skill-recall-dup`), NOT the detector-input names. The -// SDK matches `wanted_set.contains(&f.kind)` for pattern-derived findings and -// also keys `tool-output-bloat` / `ghost-surface` / `tool-call-pattern` off -// the same set, so this list has to use the finding-kind spelling on every -// row. -const PATTERN_KINDS: &[&str] = &[ - "retry-loop", - "failure-run", - "cancellation-run", - "compaction-loss", - "edit-revert", - "edit-heavy", - "skill-recall-dup", - "skill-pruning-protection", - "system-prompt-tax", - "ghost-surface", - "tool-output-bloat", - "tool-call-pattern", -]; - -fn resolve_pattern_selection(raw: &str) -> Result, String> { - if raw.is_empty() { - return Ok(PATTERN_KINDS.iter().map(|s| (*s).to_string()).collect()); - } - let mut out: Vec = Vec::new(); - for piece in raw.split(',').map(|s| s.trim()).filter(|s| !s.is_empty()) { - if !PATTERN_KINDS.contains(&piece) { - return Err(format!( - "unknown --patterns value \"{}\". Valid: {}", - piece, - PATTERN_KINDS.join(", ") - )); - } - if !out.iter().any(|s| s == piece) { - out.push(piece.to_string()); - } - } - if out.is_empty() { - return Ok(PATTERN_KINDS.iter().map(|s| (*s).to_string()).collect()); - } - Ok(out) -} - -pub fn run(globals: &GlobalArgs, args: HotspotsArgs) -> i32 { - match run_inner(globals, args) { - Ok(code) => code, - Err(err) => report_error(&err, globals), - } -} - -fn run_inner(globals: &GlobalArgs, args: HotspotsArgs) -> anyhow::Result { - // The TS surface treats `--session` (no value) as "drop into the - // per-session aggregate / gap report." That view weaves session - // relationships, tool-result chronology, and per-session attribution — - // none of which the SDK exposes yet. Keep it a clear stub. - if matches!(args.session.as_deref(), Some("")) { - eprintln!( - "burn: per-session aggregate view (`--session` with no id) is not yet implemented in the Rust port. Pass a session id to filter the standard hotspots view." - ); - return Ok(2); - } - if args.explain_drift { - eprintln!( - "burn: --explain-drift is not yet implemented in the Rust port (relationship-drift query verb hasn't landed in relayburn-sdk yet)." - ); - return Ok(2); - } - - // `--findings` standalone means "render findings unified view"; pin it - // to `--patterns` (all detectors) so the resolver below sees a value. - let patterns_arg: Option<&str> = if args.patterns.is_some() { - args.patterns.as_deref() - } else if args.findings { - Some("") - } else { - None - }; - let patterns_selection: Option> = match patterns_arg { - None => None, - Some(raw) => match resolve_pattern_selection(raw) { - Ok(sel) => Some(sel), - Err(msg) => { - eprintln!("burn: {msg}"); - return Ok(2); - } - }, - }; - - let group_by = match args.group_by.as_deref() { - None => None, - Some("attribution") => Some(HotspotsGroupBy::Attribution), - Some("bash") => Some(HotspotsGroupBy::Bash), - Some("bash-verb") => Some(HotspotsGroupBy::BashVerb), - Some("file") => Some(HotspotsGroupBy::File), - Some("subagent") => Some(HotspotsGroupBy::Subagent), - Some(other) => { - eprintln!( - "burn: unknown --group-by value \"{}\". Valid: attribution, bash, bash-verb, file, subagent", - other - ); - return Ok(2); - } - }; - - if group_by.is_some() && patterns_selection.is_some() { - eprintln!( - "burn: --group-by and --patterns/--findings are mutually exclusive (group-by selects an attribution rollup; patterns/findings drive the detector view)." - ); - return Ok(2); - } - - let provider_filter: Option> = args.provider.as_deref().and_then(|raw| { - let parts: Vec = raw - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(); - (!parts.is_empty()).then_some(parts) - }); - - // Open + ingest. We open the handle locally so ingest sees the same - // sealed `RELAYBURN_HOME` the verb call does. - let ledger_home = globals.ledger_path.clone(); - let progress = TaskProgress::new(globals, "hotspots"); - let opts = match &ledger_home { - Some(h) => LedgerOpenOptions::with_home(h), - None => LedgerOpenOptions::default(), - }; - progress.set_task("opening ledger"); - let mut handle = Ledger::open(opts)?; - - // Read-verb default: skip the pre-query sweep (see `burn summary` / - // `burn compare`). `--ingest` opts back into a one-off freshen; otherwise - // go straight to the query and let `burn ingest --watch` / the Claude Stop - // hook keep the ledger current out of band. - if args.ingest { - progress.set_task("refreshing ledger"); - let raw_opts = progress.ingest_options(ledger_home.clone()); - ingest_all(handle.raw_mut(), &raw_opts)?; - } - drop(handle); - - let session_filter = match args.session.as_deref() { - Some(s) if !s.is_empty() => Some(s.to_string()), - _ => None, - }; - - progress.set_task("analyzing hotspots"); - let result = sdk_hotspots(HotspotsOptions { - session: session_filter, - project: args.project.clone(), - since: args.since.clone(), - group_by, - patterns: patterns_selection, - workflow: args.workflow.clone(), - provider: provider_filter, - ledger_home, - })?; - progress.finish_and_clear(); - - if globals.json { - emit_json(&result)?; - return Ok(0); - } - let limit = if args.all { usize::MAX } else { DEFAULT_TOP_N }; - emit_human(&result, limit, args.findings, args.rank_by); - Ok(0) -} - -fn emit_json(result: &HotspotsResult) -> std::io::Result<()> { - let mut value = hotspots_result_to_json(result); - coerce_whole_f64_to_int(&mut value); - render_json(&value) -} - -fn hotspots_result_to_json(result: &HotspotsResult) -> Value { - match result { - HotspotsResult::Attribution(a) => attribution_to_json(a), - HotspotsResult::Bash { - rows, - refused, - refusal_reason, - } => json!({ - "rows": rows.iter().map(bash_to_json).collect::>(), - "refused": refused, - "refusalReason": refusal_reason, - }), - HotspotsResult::BashVerb { - rows, - refused, - refusal_reason, - } => json!({ - "rows": rows.iter().map(bash_verb_to_json).collect::>(), - "refused": refused, - "refusalReason": refusal_reason, - }), - HotspotsResult::File { - rows, - refused, - refusal_reason, - } => json!({ - "rows": rows.iter().map(file_to_json).collect::>(), - "refused": refused, - "refusalReason": refusal_reason, - }), - HotspotsResult::Subagent { - rows, - refused, - refusal_reason, - } => json!({ - "rows": rows.iter().map(subagent_to_json).collect::>(), - "refused": refused, - "refusalReason": refusal_reason, - }), - HotspotsResult::Findings { findings, summary } => json!({ - "findings": findings, - "summary": summary, - }), - } -} - -fn attribution_to_json(a: &HotspotsAttributionResult) -> Value { - let mut out = Map::new(); - out.insert("turnsAnalyzed".into(), json!(a.turns_analyzed)); - out.insert("grandTotal".into(), json!(a.grand_total)); - out.insert("attributedTotal".into(), json!(a.attributed_total)); - out.insert("unattributedTotal".into(), json!(a.unattributed_total)); - out.insert("attributionDegraded".into(), json!(a.attribution_degraded)); - out.insert( - "sessions".into(), - Value::Array(a.sessions.iter().map(session_total_to_json).collect()), - ); - out.insert( - "files".into(), - Value::Array(a.files.iter().map(file_to_json).collect()), - ); - out.insert( - "bashVerbs".into(), - Value::Array(a.bash_verbs.iter().map(bash_verb_to_json).collect()), - ); - out.insert( - "bash".into(), - Value::Array(a.bash.iter().map(bash_to_json).collect()), - ); - out.insert( - "subagents".into(), - Value::Array(a.subagents.iter().map(subagent_to_json).collect()), - ); - out.insert( - "mcpServers".into(), - Value::Array(a.mcp_servers.iter().map(mcp_server_to_json).collect()), - ); - out.insert( - "fidelity".into(), - json!({ - "analyzed": a.fidelity.analyzed, - "excluded": a.fidelity.excluded, - "summary": reorder_fidelity_summary(&a.fidelity.summary), - "refused": a.fidelity.refused, - }), - ); - if let Some(refused) = a.refused { - out.insert("refused".into(), json!(refused)); - } - if let Some(reason) = a.refusal_reason.as_ref() { - out.insert("refusalReason".into(), json!(reason)); - } - Value::Object(out) -} - -fn session_total_to_json(s: &HotspotsSessionTotal) -> Value { - json!({ - "sessionId": s.session_id, - "grandCost": s.grand_cost, - "attributedCost": s.attributed_cost, - "unattributedCost": s.unattributed_cost, - "attributionMethod": attribution_method_key(s.attribution_method), - }) -} - -/// Re-order the SDK-emitted fidelity summary so the JSON keys match the -/// TS-CLI snapshot ordering. The SDK builds `byClass` / -/// `byGranularity` / `missingCoverage` from `HashMap`s so iteration -/// order is non-deterministic; we reach into the `Value`, pull out the -/// numbers, and reassemble the object in the canonical order the TS -/// implementation uses (which is also the iteration order of the -/// upstream enum). -fn reorder_fidelity_summary(summary: &Value) -> Value { - use serde_json::Map; - let Some(obj) = summary.as_object() else { - return summary.clone(); - }; - let mut out = Map::new(); - out.insert( - "total".into(), - obj.get("total").cloned().unwrap_or(json!(0)), - ); - - let mut by_class = Map::new(); - let class_block = obj.get("byClass").and_then(|v| v.as_object()); - for key in [ - "full", - "usage-only", - "aggregate-only", - "cost-only", - "partial", - ] { - let v = class_block - .and_then(|m| m.get(key)) - .cloned() - .unwrap_or(json!(0)); - by_class.insert(key.to_string(), v); - } - out.insert("byClass".into(), Value::Object(by_class)); - - let mut by_granularity = Map::new(); - let gran_block = obj.get("byGranularity").and_then(|v| v.as_object()); - for key in [ - "per-turn", - "per-message", - "per-session-aggregate", - "cost-only", - ] { - let v = gran_block - .and_then(|m| m.get(key)) - .cloned() - .unwrap_or(json!(0)); - by_granularity.insert(key.to_string(), v); - } - out.insert("byGranularity".into(), Value::Object(by_granularity)); - - let mut missing = Map::new(); - let missing_block = obj.get("missingCoverage").and_then(|v| v.as_object()); - for key in [ - "hasInputTokens", - "hasOutputTokens", - "hasReasoningTokens", - "hasCacheReadTokens", - "hasCacheCreateTokens", - "hasToolCalls", - "hasToolResultEvents", - "hasSessionRelationships", - "hasRawContent", - ] { - let v = missing_block - .and_then(|m| m.get(key)) - .cloned() - .unwrap_or(json!(0)); - missing.insert(key.to_string(), v); - } - out.insert("missingCoverage".into(), Value::Object(missing)); - out.insert( - "unknown".into(), - obj.get("unknown").cloned().unwrap_or(json!(0)), - ); - Value::Object(out) -} - -fn attribution_method_key(m: AttributionMethod) -> &'static str { - match m { - AttributionMethod::Sized => "sized", - AttributionMethod::EvenSplit => "even-split", - } -} - -fn file_to_json(f: &FileAggregation) -> Value { - json!({ - "path": f.path, - "toolCallCount": f.tool_call_count, - "initialTokens": f.initial_tokens, - "persistenceTokens": f.persistence_tokens, - "ridingTurns": f.riding_turns, - "totalCost": f.total_cost, - "firstEmitTs": f.first_emit_ts, - "firstEmitTurnIndex": f.first_emit_turn_index, - "totalOutputBytes": f.total_output_bytes, - "maxOutputBytes": f.max_output_bytes, - "truncatedCount": f.truncated_count, - }) -} - -fn bash_to_json(b: &BashAggregation) -> Value { - let mut out = Map::new(); - out.insert("argsHash".into(), json!(b.args_hash)); - if let Some(c) = &b.command { - out.insert("command".into(), json!(c)); - } - out.insert("callCount".into(), json!(b.call_count)); - out.insert("totalCost".into(), json!(b.total_cost)); - out.insert("initialTokens".into(), json!(b.initial_tokens)); - out.insert("persistenceTokens".into(), json!(b.persistence_tokens)); - out.insert("totalOutputBytes".into(), json!(b.total_output_bytes)); - out.insert("maxOutputBytes".into(), json!(b.max_output_bytes)); - out.insert("truncatedCount".into(), json!(b.truncated_count)); - Value::Object(out) -} - -fn bash_verb_to_json(b: &BashVerbAggregation) -> Value { - json!({ - "verb": b.verb, - "callCount": b.call_count, - "distinctCommands": b.distinct_commands, - "totalCost": b.total_cost, - "initialTokens": b.initial_tokens, - "persistenceTokens": b.persistence_tokens, - "avgPersistenceTurns": b.avg_persistence_turns, - "topExamples": b.top_examples, - "totalOutputBytes": b.total_output_bytes, - "maxOutputBytes": b.max_output_bytes, - "truncatedCount": b.truncated_count, - }) -} - -fn subagent_to_json(s: &SubagentAggregation) -> Value { - json!({ - "subagentType": s.subagent_type, - "callCount": s.call_count, - "totalCost": s.total_cost, - "initialTokens": s.initial_tokens, - "persistenceTokens": s.persistence_tokens, - "totalOutputBytes": s.total_output_bytes, - "maxOutputBytes": s.max_output_bytes, - "truncatedCount": s.truncated_count, - }) -} - -fn mcp_server_to_json(m: &McpServerAggregation) -> Value { - json!({ - "server": m.server, - "callCount": m.call_count, - "initialTokens": m.initial_tokens, - "persistenceTokens": m.persistence_tokens, - "ridingTurns": m.riding_turns, - "totalCost": m.total_cost, - "topTools": m.top_tools, - }) -} - -// ---------- human rendering ---------- - -fn emit_human(result: &HotspotsResult, limit: usize, findings_view: bool, rank_by: RankBy) { +pub(super) fn emit_human( + result: &HotspotsResult, + limit: usize, + findings_view: bool, + rank_by: RankBy, +) { match result { HotspotsResult::Attribution(a) => emit_human_attribution(a, limit, rank_by), // The single-axis group_by surfaces aren't yet tied to a golden diff --git a/crates/relayburn-cli/src/commands/hotspots/json.rs b/crates/relayburn-cli/src/commands/hotspots/json.rs new file mode 100644 index 00000000..06d2082e --- /dev/null +++ b/crates/relayburn-cli/src/commands/hotspots/json.rs @@ -0,0 +1,280 @@ +//! JSON serialization for `burn hotspots` results. + +use relayburn_sdk::{ + AttributionMethod, BashAggregation, BashVerbAggregation, FileAggregation, + HotspotsAttributionResult, HotspotsResult, HotspotsSessionTotal, McpServerAggregation, + SubagentAggregation, +}; +use serde_json::{json, Map, Value}; + +use crate::render::format::coerce_whole_f64_to_int; +use crate::render::json::render_json; + +pub(super) fn emit_json(result: &HotspotsResult) -> std::io::Result<()> { + let mut value = hotspots_result_to_json(result); + coerce_whole_f64_to_int(&mut value); + render_json(&value) +} + +pub(super) fn hotspots_result_to_json(result: &HotspotsResult) -> Value { + match result { + HotspotsResult::Attribution(a) => attribution_to_json(a), + HotspotsResult::Bash { + rows, + refused, + refusal_reason, + } => json!({ + "rows": rows.iter().map(bash_to_json).collect::>(), + "refused": refused, + "refusalReason": refusal_reason, + }), + HotspotsResult::BashVerb { + rows, + refused, + refusal_reason, + } => json!({ + "rows": rows.iter().map(bash_verb_to_json).collect::>(), + "refused": refused, + "refusalReason": refusal_reason, + }), + HotspotsResult::File { + rows, + refused, + refusal_reason, + } => json!({ + "rows": rows.iter().map(file_to_json).collect::>(), + "refused": refused, + "refusalReason": refusal_reason, + }), + HotspotsResult::Subagent { + rows, + refused, + refusal_reason, + } => json!({ + "rows": rows.iter().map(subagent_to_json).collect::>(), + "refused": refused, + "refusalReason": refusal_reason, + }), + HotspotsResult::Findings { findings, summary } => json!({ + "findings": findings, + "summary": summary, + }), + } +} + +pub(super) fn attribution_to_json(a: &HotspotsAttributionResult) -> Value { + let mut out = Map::new(); + out.insert("turnsAnalyzed".into(), json!(a.turns_analyzed)); + out.insert("grandTotal".into(), json!(a.grand_total)); + out.insert("attributedTotal".into(), json!(a.attributed_total)); + out.insert("unattributedTotal".into(), json!(a.unattributed_total)); + out.insert("attributionDegraded".into(), json!(a.attribution_degraded)); + out.insert( + "sessions".into(), + Value::Array(a.sessions.iter().map(session_total_to_json).collect()), + ); + out.insert( + "files".into(), + Value::Array(a.files.iter().map(file_to_json).collect()), + ); + out.insert( + "bashVerbs".into(), + Value::Array(a.bash_verbs.iter().map(bash_verb_to_json).collect()), + ); + out.insert( + "bash".into(), + Value::Array(a.bash.iter().map(bash_to_json).collect()), + ); + out.insert( + "subagents".into(), + Value::Array(a.subagents.iter().map(subagent_to_json).collect()), + ); + out.insert( + "mcpServers".into(), + Value::Array(a.mcp_servers.iter().map(mcp_server_to_json).collect()), + ); + out.insert( + "fidelity".into(), + json!({ + "analyzed": a.fidelity.analyzed, + "excluded": a.fidelity.excluded, + "summary": reorder_fidelity_summary(&a.fidelity.summary), + "refused": a.fidelity.refused, + }), + ); + if let Some(refused) = a.refused { + out.insert("refused".into(), json!(refused)); + } + if let Some(reason) = a.refusal_reason.as_ref() { + out.insert("refusalReason".into(), json!(reason)); + } + Value::Object(out) +} + +pub(super) fn session_total_to_json(s: &HotspotsSessionTotal) -> Value { + json!({ + "sessionId": s.session_id, + "grandCost": s.grand_cost, + "attributedCost": s.attributed_cost, + "unattributedCost": s.unattributed_cost, + "attributionMethod": attribution_method_key(s.attribution_method), + }) +} + +/// Re-order the SDK-emitted fidelity summary so the JSON keys match the +/// TS-CLI snapshot ordering. The SDK builds `byClass` / +/// `byGranularity` / `missingCoverage` from `HashMap`s so iteration +/// order is non-deterministic; we reach into the `Value`, pull out the +/// numbers, and reassemble the object in the canonical order the TS +/// implementation uses (which is also the iteration order of the +/// upstream enum). +pub(super) fn reorder_fidelity_summary(summary: &Value) -> Value { + use serde_json::Map; + let Some(obj) = summary.as_object() else { + return summary.clone(); + }; + let mut out = Map::new(); + out.insert( + "total".into(), + obj.get("total").cloned().unwrap_or(json!(0)), + ); + + let mut by_class = Map::new(); + let class_block = obj.get("byClass").and_then(|v| v.as_object()); + for key in [ + "full", + "usage-only", + "aggregate-only", + "cost-only", + "partial", + ] { + let v = class_block + .and_then(|m| m.get(key)) + .cloned() + .unwrap_or(json!(0)); + by_class.insert(key.to_string(), v); + } + out.insert("byClass".into(), Value::Object(by_class)); + + let mut by_granularity = Map::new(); + let gran_block = obj.get("byGranularity").and_then(|v| v.as_object()); + for key in [ + "per-turn", + "per-message", + "per-session-aggregate", + "cost-only", + ] { + let v = gran_block + .and_then(|m| m.get(key)) + .cloned() + .unwrap_or(json!(0)); + by_granularity.insert(key.to_string(), v); + } + out.insert("byGranularity".into(), Value::Object(by_granularity)); + + let mut missing = Map::new(); + let missing_block = obj.get("missingCoverage").and_then(|v| v.as_object()); + for key in [ + "hasInputTokens", + "hasOutputTokens", + "hasReasoningTokens", + "hasCacheReadTokens", + "hasCacheCreateTokens", + "hasToolCalls", + "hasToolResultEvents", + "hasSessionRelationships", + "hasRawContent", + ] { + let v = missing_block + .and_then(|m| m.get(key)) + .cloned() + .unwrap_or(json!(0)); + missing.insert(key.to_string(), v); + } + out.insert("missingCoverage".into(), Value::Object(missing)); + out.insert( + "unknown".into(), + obj.get("unknown").cloned().unwrap_or(json!(0)), + ); + Value::Object(out) +} + +pub(super) fn attribution_method_key(m: AttributionMethod) -> &'static str { + match m { + AttributionMethod::Sized => "sized", + AttributionMethod::EvenSplit => "even-split", + } +} + +pub(super) fn file_to_json(f: &FileAggregation) -> Value { + json!({ + "path": f.path, + "toolCallCount": f.tool_call_count, + "initialTokens": f.initial_tokens, + "persistenceTokens": f.persistence_tokens, + "ridingTurns": f.riding_turns, + "totalCost": f.total_cost, + "firstEmitTs": f.first_emit_ts, + "firstEmitTurnIndex": f.first_emit_turn_index, + "totalOutputBytes": f.total_output_bytes, + "maxOutputBytes": f.max_output_bytes, + "truncatedCount": f.truncated_count, + }) +} + +pub(super) fn bash_to_json(b: &BashAggregation) -> Value { + let mut out = Map::new(); + out.insert("argsHash".into(), json!(b.args_hash)); + if let Some(c) = &b.command { + out.insert("command".into(), json!(c)); + } + out.insert("callCount".into(), json!(b.call_count)); + out.insert("totalCost".into(), json!(b.total_cost)); + out.insert("initialTokens".into(), json!(b.initial_tokens)); + out.insert("persistenceTokens".into(), json!(b.persistence_tokens)); + out.insert("totalOutputBytes".into(), json!(b.total_output_bytes)); + out.insert("maxOutputBytes".into(), json!(b.max_output_bytes)); + out.insert("truncatedCount".into(), json!(b.truncated_count)); + Value::Object(out) +} + +pub(super) fn bash_verb_to_json(b: &BashVerbAggregation) -> Value { + json!({ + "verb": b.verb, + "callCount": b.call_count, + "distinctCommands": b.distinct_commands, + "totalCost": b.total_cost, + "initialTokens": b.initial_tokens, + "persistenceTokens": b.persistence_tokens, + "avgPersistenceTurns": b.avg_persistence_turns, + "topExamples": b.top_examples, + "totalOutputBytes": b.total_output_bytes, + "maxOutputBytes": b.max_output_bytes, + "truncatedCount": b.truncated_count, + }) +} + +pub(super) fn subagent_to_json(s: &SubagentAggregation) -> Value { + json!({ + "subagentType": s.subagent_type, + "callCount": s.call_count, + "totalCost": s.total_cost, + "initialTokens": s.initial_tokens, + "persistenceTokens": s.persistence_tokens, + "totalOutputBytes": s.total_output_bytes, + "maxOutputBytes": s.max_output_bytes, + "truncatedCount": s.truncated_count, + }) +} + +pub(super) fn mcp_server_to_json(m: &McpServerAggregation) -> Value { + json!({ + "server": m.server, + "callCount": m.call_count, + "initialTokens": m.initial_tokens, + "persistenceTokens": m.persistence_tokens, + "ridingTurns": m.riding_turns, + "totalCost": m.total_cost, + "topTools": m.top_tools, + }) +} diff --git a/crates/relayburn-cli/src/commands/hotspots/mod.rs b/crates/relayburn-cli/src/commands/hotspots/mod.rs new file mode 100644 index 00000000..4d8a3fe2 --- /dev/null +++ b/crates/relayburn-cli/src/commands/hotspots/mod.rs @@ -0,0 +1,294 @@ +//! `burn hotspots` — surface high-cost / high-overhead hotspots from +//! the ledger. +//! +//! Thin presenter over [`relayburn_sdk::hotspots`]. Mirrors +//! `packages/cli/src/commands/hotspots.ts` for the default attribution +//! flow that drives the golden snapshots; the broader TS surface +//! (`--patterns`, `--findings`, `--session` per-session view, +//! `--provider`, `--workflow`) is enumerated as flag wiring + a +//! stub-mode error path. +//! +//! ## Wiring +//! +//! 1. Open a [`relayburn_sdk::LedgerHandle`] honoring `--ledger-path` / +//! `RELAYBURN_HOME`. +//! 2. Run [`relayburn_sdk::ingest_all`] with TTY-only progress. +//! 3. Call [`relayburn_sdk::hotspots`] (verb-form) with the resolved +//! [`relayburn_sdk::HotspotsOptions`]. The SDK enforces the coverage +//! gate, picks the `Sized` vs `EvenSplit` attribution method per +//! session, and emits the discriminated union; for the default flow +//! we expect [`relayburn_sdk::HotspotsResult::Attribution`] and +//! unwrap that branch. +//! 4. Render JSON or human format. JSON output drops the `kind` +//! discriminator and emits the inner `HotspotsAttributionResult` +//! shape directly (TS contract). + +use clap::{Args, ValueEnum}; +use relayburn_sdk::{ + hotspots as sdk_hotspots, ingest_all, HotspotsGroupBy, HotspotsOptions, Ledger, + LedgerOpenOptions, +}; + +use crate::cli::GlobalArgs; +use crate::render::error::report_error; +use crate::render::progress::TaskProgress; + +mod human; +mod json; + +use human::*; +use json::*; + +const DEFAULT_TOP_N: usize = 10; + +/// Per-command flags for `burn hotspots`. Mirrors the TS surface in +/// `packages/cli/src/commands/hotspots.ts`. +#[derive(Debug, Clone, Args)] +pub struct HotspotsArgs { + /// Slice the ledger to events at or after ``. ISO timestamp + /// or relative range (`24h`, `7d`, `4w`, `2m`). + #[arg(long, value_name = "WHEN")] + pub since: Option, + + /// Restrict to a single project. + #[arg(long, value_name = "PROJECT")] + pub project: Option, + + /// Restrict to a single session id (or pass without a value to drop + /// into the per-session attribution view). + #[arg(long, value_name = "SESSION_ID", num_args = 0..=1, default_missing_value = "")] + pub session: Option, + + /// Filter by enrichment workflow id. + #[arg(long, value_name = "WORKFLOW_ID")] + pub workflow: Option, + + /// Provider filter (CSV of provider names; case-insensitive). + #[arg(long, value_name = "PROVIDERS")] + pub provider: Option, + + /// Show all rows in human mode instead of capping at the default + /// top-N (10). + #[arg(long)] + pub all: bool, + + /// Group by a single dimension. Defaults to the full attribution + /// view; pass `bash`, `bash-verb`, `file`, or `subagent` to focus + /// a single rollup. + #[arg(long = "group-by", value_name = "DIM")] + pub group_by: Option, + + /// Comma-separated hotspot-pattern detectors to run instead of the + /// attribution view. Pass without a value to enable every detector. + #[arg(long, value_name = "PATTERNS", num_args = 0..=1, default_missing_value = "")] + pub patterns: Option, + + /// Render the unified `findings` view rather than the per-detector + /// summary. Implies `--patterns` if it isn't already set. + #[arg(long)] + pub findings: bool, + + /// Surface session relationship drift on top of the default attribution + /// view. Currently a stub in the Rust port — the relationship drift + /// query verb is not yet exposed by the SDK. + #[arg(long = "explain-drift")] + pub explain_drift: bool, + + /// Ranking dimension for the per-tool tables (files, bash, bash verbs, + /// subagents). `cost` (default) keeps the historical USD-descending + /// order; `bytes` sorts by `totalOutputBytes` so blowouts that get + /// truncated to ~0 tokens still surface (#436). JSON output is + /// unaffected — both rankings ship every field; downstream consumers + /// pick their own sort. + #[arg(long = "rank-by", value_name = "DIM", value_enum, default_value_t = RankBy::Cost)] + pub rank_by: RankBy, + + /// Run a pre-query ingest sweep so hotspots reflects freshly appended + /// sessions. Off by default: `hotspots` is a read verb, and a full-store + /// sweep re-stats every session file under every harness store — seconds + /// on a large ledger. Keep the ledger current out of band with + /// `burn ingest --watch` (or the Claude Stop hook); pass `--ingest` only + /// for a one-off freshen. Mirrors `burn summary --ingest`. + #[arg(long = "ingest")] + pub ingest: bool, +} + +/// Sort dimension for the per-tool human-mode tables. Mirrors `--rank-by`. +/// `ValueEnum` so clap validates at parse time — invalid values fail before +/// any ingest work runs, in both human and `--json` modes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum RankBy { + Cost, + Bytes, +} + +// Detector kinds the SDK's `run_hotspots_findings` filter expects. These are +// the *finding* kind strings emitted by `WasteFinding.kind` (e.g. +// `compaction-loss`, `skill-recall-dup`), NOT the detector-input names. The +// SDK matches `wanted_set.contains(&f.kind)` for pattern-derived findings and +// also keys `tool-output-bloat` / `ghost-surface` / `tool-call-pattern` off +// the same set, so this list has to use the finding-kind spelling on every +// row. +const PATTERN_KINDS: &[&str] = &[ + "retry-loop", + "failure-run", + "cancellation-run", + "compaction-loss", + "edit-revert", + "edit-heavy", + "skill-recall-dup", + "skill-pruning-protection", + "system-prompt-tax", + "ghost-surface", + "tool-output-bloat", + "tool-call-pattern", +]; + +fn resolve_pattern_selection(raw: &str) -> Result, String> { + if raw.is_empty() { + return Ok(PATTERN_KINDS.iter().map(|s| (*s).to_string()).collect()); + } + let mut out: Vec = Vec::new(); + for piece in raw.split(',').map(|s| s.trim()).filter(|s| !s.is_empty()) { + if !PATTERN_KINDS.contains(&piece) { + return Err(format!( + "unknown --patterns value \"{}\". Valid: {}", + piece, + PATTERN_KINDS.join(", ") + )); + } + if !out.iter().any(|s| s == piece) { + out.push(piece.to_string()); + } + } + if out.is_empty() { + return Ok(PATTERN_KINDS.iter().map(|s| (*s).to_string()).collect()); + } + Ok(out) +} + +pub fn run(globals: &GlobalArgs, args: HotspotsArgs) -> i32 { + match run_inner(globals, args) { + Ok(code) => code, + Err(err) => report_error(&err, globals), + } +} + +fn run_inner(globals: &GlobalArgs, args: HotspotsArgs) -> anyhow::Result { + // The TS surface treats `--session` (no value) as "drop into the + // per-session aggregate / gap report." That view weaves session + // relationships, tool-result chronology, and per-session attribution — + // none of which the SDK exposes yet. Keep it a clear stub. + if matches!(args.session.as_deref(), Some("")) { + eprintln!( + "burn: per-session aggregate view (`--session` with no id) is not yet implemented in the Rust port. Pass a session id to filter the standard hotspots view." + ); + return Ok(2); + } + if args.explain_drift { + eprintln!( + "burn: --explain-drift is not yet implemented in the Rust port (relationship-drift query verb hasn't landed in relayburn-sdk yet)." + ); + return Ok(2); + } + + // `--findings` standalone means "render findings unified view"; pin it + // to `--patterns` (all detectors) so the resolver below sees a value. + let patterns_arg: Option<&str> = if args.patterns.is_some() { + args.patterns.as_deref() + } else if args.findings { + Some("") + } else { + None + }; + let patterns_selection: Option> = match patterns_arg { + None => None, + Some(raw) => match resolve_pattern_selection(raw) { + Ok(sel) => Some(sel), + Err(msg) => { + eprintln!("burn: {msg}"); + return Ok(2); + } + }, + }; + + let group_by = match args.group_by.as_deref() { + None => None, + Some("attribution") => Some(HotspotsGroupBy::Attribution), + Some("bash") => Some(HotspotsGroupBy::Bash), + Some("bash-verb") => Some(HotspotsGroupBy::BashVerb), + Some("file") => Some(HotspotsGroupBy::File), + Some("subagent") => Some(HotspotsGroupBy::Subagent), + Some(other) => { + eprintln!( + "burn: unknown --group-by value \"{}\". Valid: attribution, bash, bash-verb, file, subagent", + other + ); + return Ok(2); + } + }; + + if group_by.is_some() && patterns_selection.is_some() { + eprintln!( + "burn: --group-by and --patterns/--findings are mutually exclusive (group-by selects an attribution rollup; patterns/findings drive the detector view)." + ); + return Ok(2); + } + + let provider_filter: Option> = args.provider.as_deref().and_then(|raw| { + let parts: Vec = raw + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + (!parts.is_empty()).then_some(parts) + }); + + // Open + ingest. We open the handle locally so ingest sees the same + // sealed `RELAYBURN_HOME` the verb call does. + let ledger_home = globals.ledger_path.clone(); + let progress = TaskProgress::new(globals, "hotspots"); + let opts = match &ledger_home { + Some(h) => LedgerOpenOptions::with_home(h), + None => LedgerOpenOptions::default(), + }; + progress.set_task("opening ledger"); + let mut handle = Ledger::open(opts)?; + + // Read-verb default: skip the pre-query sweep (see `burn summary` / + // `burn compare`). `--ingest` opts back into a one-off freshen; otherwise + // go straight to the query and let `burn ingest --watch` / the Claude Stop + // hook keep the ledger current out of band. + if args.ingest { + progress.set_task("refreshing ledger"); + let raw_opts = progress.ingest_options(ledger_home.clone()); + ingest_all(handle.raw_mut(), &raw_opts)?; + } + drop(handle); + + let session_filter = match args.session.as_deref() { + Some(s) if !s.is_empty() => Some(s.to_string()), + _ => None, + }; + + progress.set_task("analyzing hotspots"); + let result = sdk_hotspots(HotspotsOptions { + session: session_filter, + project: args.project.clone(), + since: args.since.clone(), + group_by, + patterns: patterns_selection, + workflow: args.workflow.clone(), + provider: provider_filter, + ledger_home, + })?; + progress.finish_and_clear(); + + if globals.json { + emit_json(&result)?; + return Ok(0); + } + let limit = if args.all { usize::MAX } else { DEFAULT_TOP_N }; + emit_human(&result, limit, args.findings, args.rank_by); + Ok(0) +} diff --git a/crates/relayburn-cli/src/commands/summary.rs b/crates/relayburn-cli/src/commands/summary.rs deleted file mode 100644 index f8c6e2c5..00000000 --- a/crates/relayburn-cli/src/commands/summary.rs +++ /dev/null @@ -1,1515 +0,0 @@ -//! `burn summary` — aggregate session usage and cost. -//! -//! Thin presenter over the `relayburn_sdk` query helpers. Mirrors the TS -//! `packages/cli/src/commands/summary.ts` surface: grouped model/provider -//! summaries, tool attribution, subagent views, relationship rollups, workflow -//! / agent / provider filters, and the optional quality footer. -//! -//! ## Wiring -//! -//! 1. Open a [`relayburn_sdk::LedgerHandle`] honoring `--ledger-path` / -//! `RELAYBURN_HOME`. -//! 2. Optionally run [`relayburn_sdk::ingest_all`] against the same handle, -//! gated on `--ingest`. `summary` is a read verb, and a pre-query sweep -//! re-stats every session file under every harness store — seconds on a -//! large OpenCode store (tens of thousands of sessions) — so it is -//! **off by default**. The steady-state setup keeps the ledger fresh out -//! of band: `burn ingest --watch` once per host, or the Claude Stop hook -//! (`burn ingest --hook claude`) firing each turn. Pass `--ingest` for a -//! one-off freshen; the human banner then leads with -//! `ingested N new sessions (+M turns)` and a TTY gets a stderr spinner. -//! When the sweep is skipped, an empty [`relayburn_sdk::IngestReport`] -//! keeps the banner / JSON `ingest` block byte-identical to a no-op sweep -//! (`ingested 0 new sessions`). See `burn compare` for the same decision. -//! 3. Lower CLI flags into [`relayburn_sdk::SummaryReportOptions`] and call -//! the SDK-owned `summary_report` verb. -//! 4. Render the typed report as JSON or human output. - -use std::collections::{BTreeMap, BTreeSet}; - -use clap::Args; -use relayburn_sdk::{ - ingest_all, summary_fidelity_summary_to_value, summary_replacement_savings_to_value, - CostBreakdown, CoverageField, Enrichment, FidelityClass, FidelitySummary, Ledger, LedgerHandle, - LedgerOpenOptions, OutcomeLabel, QualityResult, RelationshipType, StopReasonCounts, - SubagentCounts, SubagentTreeNode, SubagentTypeStats, SummaryByToolReport, SummaryGroupBy, - SummaryGroupedReport, SummaryRelationshipReport, SummaryReport, SummaryReportMode, - SummaryReportOptions, SummarySubagentTreeReport, UsageCostAggregateRow, -}; -use serde_json::{json, Map, Value}; - -use crate::cli::GlobalArgs; -use crate::render::error::report_error; -use crate::render::format::{coerce_whole_f64_to_int, format_uint, format_usd, render_table}; -use crate::render::json::render_json; -use crate::render::progress::TaskProgress; - -/// Per-command flags for `burn summary`. Mirrors the TS surface in -/// `packages/cli/src/commands/summary.ts` so a TS user can carry their -/// muscle memory across. -#[derive(Debug, Clone, Args)] -pub struct SummaryArgs { - /// Slice the ledger to events at or after ``. Accepts either an - /// ISO timestamp or a relative range (`24h`, `7d`, `4w`, `2m`). - #[arg(long, value_name = "WHEN")] - pub since: Option, - - /// Restrict to a single project (matches `project` or `projectKey`). - #[arg(long, value_name = "PROJECT")] - pub project: Option, - - /// Restrict to a single session id. - #[arg(long, value_name = "SESSION_ID")] - pub session: Option, - - /// Group by effective provider instead of model. - #[arg(long = "by-provider")] - pub by_provider: bool, - - /// Group by tool, attributing each turn's ingest cost to the prior - /// turn's `tool_use` blocks. Emits a `byTool` table; mutually - /// exclusive with `--by-provider` / the subagent flags. - #[arg(long = "by-tool")] - pub by_tool: bool, - - /// Bucket by `subagent.subagentType`. Mutually exclusive with the - /// other group-by flags. - #[arg(long = "by-subagent-type")] - pub by_subagent_type: bool, - - /// Bucket by `SessionRelationshipRecord.relationshipType` (or pass - /// `subagent` to drill into the subagent leaf). Mutually exclusive - /// with `--subagent-tree`. - #[arg(long = "by-relationship", value_name = "MODE", num_args = 0..=1, default_missing_value = "")] - pub by_relationship: Option, - - /// Render the subagent spawn tree for a session id. Passing the flag - /// without a value uses `--session`. - #[arg(long = "subagent-tree", value_name = "SESSION_ID", num_args = 0..=1, default_missing_value = "")] - pub subagent_tree: Option, - - /// Restrict the subagent tree / relationship views to a single agent - /// id. - #[arg(long, value_name = "AGENT_ID")] - pub agent: Option, - - /// Filter by enrichment workflow id. - #[arg(long, value_name = "WORKFLOW_ID")] - pub workflow: Option, - - /// Filter by folded enrichment tag. Repeatable; every tag must match. - #[arg(long = "tag", value_name = "K=V")] - pub tag: Vec, - - /// Group totals by a folded enrichment tag value. - #[arg(long = "group-by-tag", value_name = "KEY")] - pub group_by_tag: Option, - - /// Provider filter (CSV of provider names; case-insensitive). - #[arg(long, value_name = "PROVIDERS")] - pub provider: Option, - - /// Append a quality summary (one-shot rate, completion outcomes). - #[arg(long)] - pub quality: bool, - - /// Accepted for TS CLI flag parity; a no-op against the Rust SDK, - /// which is SQLite-native and has no archive layer to bypass. - #[arg(long = "no-archive")] - pub no_archive: bool, - - /// Run a pre-query ingest sweep so the summary leads with freshly - /// appended sessions. Off by default: `summary` is a read verb, and a - /// full-store sweep re-stats every session file under every harness - /// store — seconds on a large ledger. Keep the ledger current out of - /// band with `burn ingest --watch` (or the Claude Stop hook); pass - /// `--ingest` only for a one-off freshen. - #[arg(long = "ingest")] - pub ingest: bool, - - /// Emit a time-series instead of a single total: bucket the `--since` - /// window into fixed-width windows and report per-bucket cost/usage. - /// Duration grammar: `30s`, `5m` (minutes), `1h`, `12h`, `1d`, `7d`. - /// Only valid for the default grouped (`byModel`/`--by-provider`) summary. - #[arg(long, value_name = "DURATION")] - pub bucket: Option, -} - -pub fn run(globals: &GlobalArgs, args: SummaryArgs) -> i32 { - match run_inner(globals, args) { - Ok(code) => code, - Err(err) => report_error(&err, globals), - } -} - -fn run_inner(globals: &GlobalArgs, args: SummaryArgs) -> anyhow::Result { - // Mode exclusivity — mirror the TS CLI's stderr+exit2 contract so a - // mis-typed combination of flags produces a clear message rather than - // silently dropping one. - if args.by_tool - && (args.by_provider - || args.by_subagent_type - || args.by_relationship.is_some() - || args.subagent_tree.is_some() - || args.group_by_tag.is_some()) - { - eprintln!( - "burn: --by-tool cannot be combined with --by-provider/--by-subagent-type/--by-relationship/--subagent-tree/--group-by-tag" - ); - return Ok(2); - } - if args.by_provider - && (args.by_subagent_type - || args.by_relationship.is_some() - || args.subagent_tree.is_some() - || args.group_by_tag.is_some()) - { - eprintln!( - "burn: --by-provider cannot be combined with --by-subagent-type/--by-relationship/--subagent-tree/--group-by-tag" - ); - return Ok(2); - } - if args.by_subagent_type - && (args.by_relationship.is_some() - || args.subagent_tree.is_some() - || args.group_by_tag.is_some()) - { - eprintln!( - "burn: --by-subagent-type cannot be combined with --by-relationship/--subagent-tree/--group-by-tag" - ); - return Ok(2); - } - if args.by_relationship.is_some() - && (args.subagent_tree.is_some() || args.group_by_tag.is_some()) - { - eprintln!("burn: --by-relationship cannot be combined with --subagent-tree/--group-by-tag"); - return Ok(2); - } - if args.subagent_tree.is_some() && args.group_by_tag.is_some() { - eprintln!("burn: --subagent-tree cannot be combined with --group-by-tag"); - return Ok(2); - } - if let Some(rel) = &args.by_relationship { - if !rel.is_empty() && rel != "subagent" { - eprintln!("burn: --by-relationship accepts only the optional value \"subagent\""); - return Ok(2); - } - } - if let Some(tag_key) = args.group_by_tag.as_deref() { - if tag_key.is_empty() { - eprintln!("burn: --group-by-tag requires a non-empty key"); - return Ok(2); - } - } - - // `--bucket` opts into a per-bucket time-series. Parse and validate it - // (including the mode/flag combinations it supports) before opening the - // ledger or running ingest, so a bad invocation fails fast. - let bucket_secs = if let Some(bucket_raw) = args.bucket.as_deref() { - if args.by_tool - || args.by_subagent_type - || args.by_relationship.is_some() - || args.subagent_tree.is_some() - || args.group_by_tag.is_some() - { - eprintln!( - "burn: --bucket is only supported with the default grouped summary or --by-provider" - ); - return Ok(2); - } - if args.quality { - eprintln!("burn: --bucket is not supported with --quality"); - return Ok(2); - } - match relayburn_sdk::parse_bucket(bucket_raw) { - Ok(secs) => Some(secs), - Err(err) => { - eprintln!("burn: {err}"); - return Ok(2); - } - } - } else { - None - }; - - let provider_filter = match parse_provider_filter(args.provider.as_deref()) { - Ok(filter) => filter, - Err(msg) => { - eprintln!("{msg}"); - return Ok(2); - } - }; - let tag_filter: Enrichment = match parse_tag_filters(&args.tag) { - Ok(filter) => filter, - Err(err) => { - eprintln!("{err}"); - return Ok(2); - } - }; - let subagent_tree_session_id = if let Some(tree_flag) = args.subagent_tree.as_deref() { - if tree_flag.is_empty() && args.session.is_none() { - eprintln!("burn: --subagent-tree requires a session id (positional or --session)"); - return Ok(2); - } - Some(if tree_flag.is_empty() { - None - } else { - Some(tree_flag.to_string()) - }) - } else { - None - }; - - // `--no-archive` is accepted for TS CLI flag parity but is a no-op: - // the Rust SDK is SQLite-native and has no archive layer to bypass. - let _ = args.no_archive; - let progress = TaskProgress::new(globals, "summary"); - - let opts = match globals.ledger_path.as_deref() { - Some(h) => LedgerOpenOptions::with_home(h), - None => LedgerOpenOptions::default(), - }; - progress.set_task("opening ledger"); - let mut handle = Ledger::open(opts).inspect_err(|_| { - progress.finish_and_clear(); - })?; - - // Read-verb default: skip the pre-query sweep (see the module doc and - // `burn compare`). `--ingest` opts back into a one-off freshen; otherwise - // an empty report keeps the banner / JSON `ingest` block identical to a - // no-op sweep without paying for the full-store stat walk. - let ingest_report = if args.ingest { - run_ingest(&mut handle, &progress, globals.ledger_path.clone()).inspect_err(|_| { - progress.finish_and_clear(); - })? - } else { - relayburn_sdk::IngestReport::empty() - }; - - let mode = if let Some(session_id) = subagent_tree_session_id { - SummaryReportMode::SubagentTree { session_id } - } else if args.by_tool { - SummaryReportMode::ByTool - } else if args.by_subagent_type { - SummaryReportMode::BySubagentType - } else if let Some(rel_flag) = args.by_relationship.as_deref() { - SummaryReportMode::ByRelationship { - subagent: rel_flag == "subagent", - } - } else { - SummaryReportMode::Grouped { - by_provider: args.by_provider, - } - }; - - let opts = SummaryReportOptions { - session: args.session, - project: args.project, - since: args.since, - workflow: args.workflow, - tags: if tag_filter.is_empty() { - None - } else { - Some(tag_filter) - }, - group_by_tag: args.group_by_tag, - agent: args.agent, - providers: provider_filter.map(|providers| providers.into_iter().collect()), - mode, - include_quality: args.quality, - ledger_home: None, - }; - - // `--bucket` switches to a per-bucket time-series of the grouped summary. - // Parsing/validation already happened above, before the ledger was opened. - if let Some(bucket_secs) = bucket_secs { - progress.set_task("building summary time-series"); - let series = handle - .summary_timeseries(opts, bucket_secs) - .inspect_err(|_| { - progress.finish_and_clear(); - })?; - progress.finish_and_clear(); - return emit_summary_timeseries(globals, &series, &ingest_report); - } - - progress.set_task("building summary"); - let report = handle.summary_report(opts).inspect_err(|_| { - progress.finish_and_clear(); - })?; - progress.finish_and_clear(); - - match report { - SummaryReport::Grouped(report) => { - emit_grouped(globals, &report, &ingest_report)?; - } - SummaryReport::ByTool(report) => { - emit_ingest_prelude(globals, &ingest_report); - return render_by_tool_report(globals, &report, &ingest_report); - } - SummaryReport::BySubagentType(report) => { - emit_ingest_prelude(globals, &ingest_report); - return render_subagent_type_report(globals, &report.stats); - } - SummaryReport::Relationship(report) => { - emit_ingest_prelude(globals, &ingest_report); - return render_relationship_report(globals, &report); - } - SummaryReport::SubagentTree(report) => { - emit_ingest_prelude(globals, &ingest_report); - return render_subagent_tree_report(globals, &report); - } - } - Ok(0) -} - -fn parse_provider_filter(raw: Option<&str>) -> Result>, &'static str> { - let Some(raw) = raw else { - return Ok(None); - }; - let providers: BTreeSet = raw - .split(',') - .map(|s| s.trim().to_ascii_lowercase()) - .filter(|s| !s.is_empty()) - .collect(); - if providers.is_empty() { - return Err("burn: --provider requires a value"); - } - Ok(Some(providers)) -} - -fn parse_tag_filters(tags: &[String]) -> anyhow::Result> { - let mut out = BTreeMap::new(); - for raw in tags { - let (key, value) = raw - .split_once('=') - .ok_or_else(|| anyhow::anyhow!("burn: --tag expects k=v, got \"{raw}\""))?; - if key.is_empty() { - anyhow::bail!("burn: --tag key must be non-empty (got \"{raw}\")"); - } - if let Some(existing) = out.get(key) { - anyhow::bail!( - "burn: duplicate --tag filter for key \"{key}\" ({existing:?} vs {value:?})" - ); - } - out.insert(key.to_string(), value.to_string()); - } - Ok(out) -} - -/// Run an ingest sweep on the open handle. -fn run_ingest( - handle: &mut LedgerHandle, - progress: &TaskProgress, - ledger_home: Option, -) -> anyhow::Result { - progress.set_task("refreshing ledger"); - let opts = progress.ingest_options(ledger_home); - ingest_all(handle.raw_mut(), &opts).inspect_err(|_| { - progress.finish_and_clear(); - }) -} - -const COVERAGE_FIELDS: [CoverageField; 5] = [ - CoverageField::Input, - CoverageField::Output, - CoverageField::Reasoning, - CoverageField::CacheRead, - CoverageField::CacheCreate, -]; - -fn cell_is_partial(c: &relayburn_sdk::FieldCoverage) -> bool { - c.known > 0 && c.missing > 0 -} - -const PARTIAL_MARK: &str = "*"; -const DASH: &str = "—"; - -fn coverage_field_label(field: CoverageField) -> &'static str { - match field { - CoverageField::Input => "input", - CoverageField::Output => "output", - CoverageField::Reasoning => "reasoning", - CoverageField::CacheRead => "cacheRead", - CoverageField::CacheCreate => "cacheCreate", - } -} - -/// Render one token-field cell. Three cases: -/// - every contributing turn reported the field → numeric value, no marker -/// - some turns reported, some didn't → numeric value + `*` -/// - no turn reported → `—` (never `0`, which -/// would falsely claim a real zero from the source) -fn coverage_cell(value: u64, c: &relayburn_sdk::FieldCoverage) -> String { - if c.known == 0 && c.missing > 0 { - return DASH.to_string(); - } - if c.known > 0 && c.missing > 0 { - return format!("{}{}", format_uint(value), PARTIAL_MARK); - } - format_uint(value) -} - -fn emit_grouped( - globals: &GlobalArgs, - report: &SummaryGroupedReport, - ingest_report: &relayburn_sdk::IngestReport, -) -> std::io::Result<()> { - if globals.json { - return emit_json(report, ingest_report); - } - emit_human(report, ingest_report); - Ok(()) -} - -fn emit_ingest_prelude(globals: &GlobalArgs, ingest_report: &relayburn_sdk::IngestReport) { - if globals.json { - return; - } - emit_human_ingest_prelude(ingest_report); -} - -fn emit_human_ingest_prelude(ingest_report: &relayburn_sdk::IngestReport) { - print!("{}", ingest_prelude_text(ingest_report)); -} - -fn ingest_prelude_text(ingest_report: &relayburn_sdk::IngestReport) -> String { - format!( - "\ningested {} new session{} (+{} turns)", - ingest_report.ingested_sessions, - if ingest_report.ingested_sessions == 1 { - "" - } else { - "s" - }, - format_uint(ingest_report.appended_turns as u64), - ) + "\n" -} - -fn emit_json( - report: &SummaryGroupedReport, - ingest_report: &relayburn_sdk::IngestReport, -) -> std::io::Result<()> { - let value = grouped_json_value(report, ingest_report); - render_json(&value) -} - -/// Render a `--bucket` time-series. JSON emits `{ bucketSeconds, buckets: [...] }` -/// (the consumer); human output is one line per bucket. -fn emit_summary_timeseries( - globals: &GlobalArgs, - series: &relayburn_sdk::SummaryTimeseries, - ingest_report: &relayburn_sdk::IngestReport, -) -> anyhow::Result { - if globals.json { - render_json(series)?; - return Ok(0); - } - emit_human_ingest_prelude(ingest_report); - if series.buckets.is_empty() { - println!("(no data in range)"); - return Ok(0); - } - for bucket in &series.buckets { - println!( - "{} {:>5} turns {:>14} tok {}", - bucket.start, - bucket.turn_count, - format_uint(bucket.total_tokens), - format_usd(bucket.total_cost.total), - ); - } - Ok(0) -} - -fn grouped_json_value( - report: &SummaryGroupedReport, - ingest_report: &relayburn_sdk::IngestReport, -) -> Value { - let key = report.group_by.json_key(); - let label_key = report.group_by.wire_str(); - - let group_rows: Vec = report - .rows - .iter() - .enumerate() - .map(|(idx, r)| { - let mut row = if report.group_by == SummaryGroupBy::Tag { - json!({ - "tag": report.tag_key.as_deref().unwrap_or(""), - "value": report.tag_values.get(idx).cloned().flatten(), - }) - } else { - json!({ - label_key: r.label, - }) - }; - let obj = row.as_object_mut().unwrap(); - obj.insert("turns".into(), json!(r.turns)); - obj.insert( - "usage".into(), - json!({ - "input": r.usage.input, - "output": r.usage.output, - "reasoning": r.usage.reasoning, - "cacheRead": r.usage.cache_read, - "cacheCreate5m": r.usage.cache_create_5m, - "cacheCreate1h": r.usage.cache_create_1h, - }), - ); - obj.insert("cost".into(), cost_breakdown_to_json(&r.cost)); - row - }) - .collect(); - - let mut payload = Map::new(); - payload.insert( - "ingest".into(), - json!({ - "ingestedSessions": ingest_report.ingested_sessions, - "appendedTurns": ingest_report.appended_turns, - }), - ); - payload.insert("turns".into(), json!(report.turn_count)); - payload.insert( - "totalCost".into(), - cost_breakdown_to_json(&report.total_cost), - ); - payload.insert(key.into(), Value::Array(group_rows)); - payload.insert( - "fidelity".into(), - json!({ - "summary": summary_fidelity_summary_to_value(&report.fidelity), - "perCell": report.per_cell_fidelity.clone(), - }), - ); - if report.replacement_savings.calls > 0 { - payload.insert( - "replacementSavings".into(), - summary_replacement_savings_to_value(&report.replacement_savings), - ); - } - payload.insert( - "stopReasons".into(), - stop_reasons_to_json(&report.stop_reasons), - ); - if !report.subagents.is_empty() { - // `subagents: {paired, orphan, total}` (issue #435). Skipped - // when both buckets are zero so the JSON shape stays compact - // for sessions that never spawned a subagent. - payload.insert( - "subagents".into(), - json!({ - "paired": report.subagents.paired, - "orphan": report.subagents.orphan, - "total": report.subagents.total(), - }), - ); - } - if let Some(quality) = report.quality.as_ref() { - payload.insert("quality".into(), json!(quality)); - } - - let mut value = Value::Object(payload); - coerce_whole_f64_to_int(&mut value); - value -} - -fn cost_breakdown_to_json(c: &CostBreakdown) -> Value { - json!({ - "model": c.model.as_ref(), - "total": c.total, - "input": c.input, - "output": c.output, - "reasoning": c.reasoning, - "cacheRead": c.cache_read, - "cacheCreate": c.cache_create, - }) -} - -fn render_by_tool_report( - globals: &GlobalArgs, - report: &SummaryByToolReport, - ingest_report: &relayburn_sdk::IngestReport, -) -> anyhow::Result { - if globals.json { - let by_tool_json: Vec = report - .rows - .iter() - .map(|item| { - let mut row = Map::new(); - row.insert("tool".into(), json!(item.tool)); - row.insert("calls".into(), json!(item.calls)); - row.insert("attributedCost".into(), json!(item.attributed_cost)); - row.insert( - "attributionMethod".into(), - json!(item.attribution_method.wire_str()), - ); - if let Some(s) = item.savings.as_ref() { - row.insert( - "savings".into(), - json!({ - "calls": s.calls, - "collapsedCalls": s.collapsed_calls, - "estimatedTokensSaved": s.estimated_tokens_saved, - }), - ); - } - Value::Object(row) - }) - .collect(); - let mut payload = json!({ - "ingest": { - "ingestedSessions": ingest_report.ingested_sessions, - "appendedTurns": ingest_report.appended_turns, - }, - "turns": report.turn_count, - "byTool": by_tool_json, - "unattributed": report.unattributed_cost, - "fidelity": { "summary": summary_fidelity_summary_to_value(&report.fidelity) }, - }); - if report.replacement_savings.calls > 0 { - payload.as_object_mut().unwrap().insert( - "replacementSavings".into(), - summary_replacement_savings_to_value(&report.replacement_savings), - ); - } - coerce_whole_f64_to_int(&mut payload); - render_json(&payload)?; - return Ok(0); - } - - let mut out = Vec::new(); - out.push(String::new()); - out.push(format!( - "turns analyzed: {}", - format_uint(report.turn_count) - )); - out.push(String::new()); - if report.rows.is_empty() { - out.push("no tool calls found for filters.".to_string()); - let mut text = out.join("\n"); - text.push('\n'); - print!("{text}"); - return Ok(0); - } - - let has_savings = report.replacement_savings.calls > 0; - let mut rows: Vec> = if has_savings { - vec![vec![ - "tool".into(), - "calls".into(), - "attributedCost".into(), - "savedTokens".into(), - ]] - } else { - vec![vec!["tool".into(), "calls".into(), "attributedCost".into()]] - }; - for item in &report.rows { - let mut row = vec![ - item.tool.clone(), - format_uint(item.calls), - format_usd(item.attributed_cost), - ]; - if has_savings { - let saved = item - .savings - .as_ref() - .map(|s| format_uint(s.estimated_tokens_saved)) - .unwrap_or_else(|| "-".to_string()); - row.push(saved); - } - rows.push(row); - } - out.push(render_table(&rows)); - out.push(String::new()); - out.push( - "attributedCost = turn N ingest cost assigned to turn N-1 tool_use blocks by user-turn byte size when available, otherwise split evenly.".to_string(), - ); - out.push(format!( - "unattributed cost (no prior tool call or non-tool user text): {}", - format_usd(report.unattributed_cost), - )); - if has_savings { - out.push(format_replacement_savings_line(&report.replacement_savings)); - } - out.push(String::new()); - print!("{}", out.join("\n")); - Ok(0) -} - -fn render_subagent_type_report( - globals: &GlobalArgs, - stats: &[SubagentTypeStats], -) -> anyhow::Result { - if globals.json { - let mut value = serde_json::to_value(stats)?; - coerce_whole_f64_to_int(&mut value); - render_json(&value)?; - return Ok(0); - } - - let mut out = Vec::new(); - out.push(String::new()); - out.push(format!( - "subagent invocations: {}", - format_uint(stats.iter().map(|s| s.invocations).sum()), - )); - out.push(String::new()); - if stats.is_empty() { - out.push(" (no subagent turns in range)".to_string()); - out.push(String::new()); - print!("{}", out.join("\n")); - return Ok(0); - } - out.push(render_subagent_stats_table(stats)); - out.push(String::new()); - print!("{}", out.join("\n")); - Ok(0) -} - -fn render_subagent_stats_table(stats: &[SubagentTypeStats]) -> String { - let mut rows = vec![vec![ - "subagentType".into(), - "invocations".into(), - "turns".into(), - "total".into(), - "median".into(), - "p95".into(), - "mean".into(), - ]]; - for s in stats { - rows.push(vec![ - s.subagent_type.clone(), - format_uint(s.invocations), - format_uint(s.turns), - format_usd(s.total_cost), - format_usd(s.median_cost), - format_usd(s.p95_cost), - format_usd(s.mean_cost), - ]); - } - render_table(&rows) -} - -const NO_RELATIONSHIPS_MESSAGE: &str = - "no SessionRelationshipRecord rows found for the matched slice; ingest a session with execution-graph wiring or run `burn state rebuild` once relationship backfill is available"; - -fn render_relationship_report( - globals: &GlobalArgs, - report: &SummaryRelationshipReport, -) -> anyhow::Result { - if !report.subagent_types.is_empty() { - return render_relationship_subagent_report(globals, report); - } - if report.relationships.is_empty() { - return render_no_relationships(globals); - } - - if globals.json { - let mut value = json!({ "relationships": report.relationships }); - coerce_whole_f64_to_int(&mut value); - render_json(&value)?; - return Ok(0); - } - - let mut out = Vec::new(); - out.push(String::new()); - out.push(format!( - "relationships: {}", - format_uint(report.relationships.iter().map(|s| s.count).sum()), - )); - out.push(String::new()); - let mut rows = vec![vec![ - "relationshipType".into(), - "count".into(), - "turnCount".into(), - "total".into(), - "median".into(), - "p95".into(), - "mean".into(), - ]]; - for s in &report.relationships { - rows.push(vec![ - s.relationship_type.wire_str().to_string(), - format_uint(s.count), - format_uint(s.turn_count), - format_usd(s.total_cost), - format_usd(s.median_cost), - format_usd(s.p95_cost), - format_usd(s.mean_cost), - ]); - } - out.push(render_table(&rows)); - out.push(String::new()); - print!("{}", out.join("\n")); - Ok(0) -} - -fn render_relationship_subagent_report( - globals: &GlobalArgs, - report: &SummaryRelationshipReport, -) -> anyhow::Result { - if report.subagent_types.is_empty() { - if globals.json { - let mut value = json!({ - "relationships": [], - "subagentTypes": [], - "message": NO_RELATIONSHIPS_MESSAGE, - }); - coerce_whole_f64_to_int(&mut value); - render_json(&value)?; - return Ok(0); - } - return render_no_relationships(globals); - } - if globals.json { - let mut value = json!({ - "relationships": report.relationships, - "subagentTypes": report.subagent_types, - }); - coerce_whole_f64_to_int(&mut value); - render_json(&value)?; - return Ok(0); - } - - let mut out = Vec::new(); - out.push(String::new()); - out.push(format!( - "subagent invocations: {}", - format_uint(report.subagent_types.iter().map(|s| s.invocations).sum()), - )); - out.push(String::new()); - let mut rows = vec![vec![ - "subagentType".into(), - "invocations".into(), - "turns".into(), - "total".into(), - "median".into(), - "p95".into(), - "mean".into(), - ]]; - for s in &report.subagent_types { - rows.push(vec![ - s.subagent_type.clone(), - format_uint(s.invocations), - format_uint(s.turns), - format_usd(s.total_cost), - format_usd(s.median_cost), - format_usd(s.p95_cost), - format_usd(s.mean_cost), - ]); - } - out.push(render_table(&rows)); - out.push(String::new()); - print!("{}", out.join("\n")); - Ok(0) -} - -fn render_no_relationships(globals: &GlobalArgs) -> anyhow::Result { - if globals.json { - render_json(&json!({ - "relationships": [], - "message": NO_RELATIONSHIPS_MESSAGE, - }))?; - } else { - println!("{NO_RELATIONSHIPS_MESSAGE}"); - } - Ok(0) -} - -fn render_subagent_tree_report( - globals: &GlobalArgs, - report: &SummarySubagentTreeReport, -) -> anyhow::Result { - if globals.json { - let root = match report.root.as_ref() { - Some(root) => serde_json::to_value(root)?, - None => Value::Null, - }; - let mut value = json!({ - "sessionId": report.session_id.as_str(), - "root": root, - }); - coerce_whole_f64_to_int(&mut value); - render_json(&value)?; - return Ok(0); - } - - let Some(root) = report.root.as_ref() else { - println!("no turns found for session {}", report.session_id); - return Ok(0); - }; - - let mut out = Vec::new(); - out.push(String::new()); - out.push(format!("session: {}", report.session_id)); - out.push(format!( - "total: {} across {} turn{}", - format_usd(root.cumulative_cost), - format_uint(root.cumulative_turns), - if root.cumulative_turns == 1 { "" } else { "s" }, - )); - out.push(String::new()); - out.extend(render_tree(root)); - out.push(String::new()); - print!("{}", out.join("\n")); - Ok(0) -} - -fn render_tree(root: &SubagentTreeNode) -> Vec { - let mut out = Vec::new(); - out.push(render_node_line(root, "")); - render_children(root, "", &mut out); - out -} - -fn render_children(node: &SubagentTreeNode, prefix: &str, out: &mut Vec) { - let n = node.children.len(); - for (i, child) in node.children.iter().enumerate() { - let is_last = i == n - 1; - let branch = if is_last { "└─ " } else { "├─ " }; - out.push(render_node_line(child, &format!("{prefix}{branch}"))); - let child_prefix = if is_last { - format!("{prefix} ") - } else { - format!("{prefix}│ ") - }; - render_children(child, &child_prefix, out); - } -} - -fn render_node_line(node: &SubagentTreeNode, indent: &str) -> String { - let relationship = if node.relationship_type != RelationshipType::Root - && node.relationship_type != RelationshipType::Subagent - { - format!(" [{}]", node.relationship_type.wire_str()) - } else { - String::new() - }; - let model = if node.models.is_empty() { - String::new() - } else { - format!(" ({})", node.models.join(", ")) - }; - format!( - "{}{}{}{} {} [{} turn{}]", - indent, - node.label, - relationship, - model, - format_usd(node.cumulative_cost), - format_uint(node.cumulative_turns), - if node.cumulative_turns == 1 { "" } else { "s" }, - ) -} - -fn emit_human(report: &SummaryGroupedReport, ingest_report: &relayburn_sdk::IngestReport) { - let mut lines: Vec = Vec::new(); - emit_human_ingest_prelude(ingest_report); - lines.push(String::new()); - - lines.push(format!( - "turns analyzed: {}", - format_uint(report.turn_count) - )); - lines.push(String::new()); - - if report.rows.is_empty() { - lines.push("no turns match the current filters.".to_string()); - let mut out = lines.join("\n"); - out.push('\n'); - print!("{}", out); - return; - } - - let header_label = if report.group_by == SummaryGroupBy::Tag { - "value" - } else { - report.group_by.wire_str() - }; - let header = vec![ - header_label.to_string(), - "turns".into(), - "input".into(), - "output".into(), - "reasoning".into(), - "cacheRead".into(), - "cacheCreate".into(), - "cost".into(), - ]; - let mut rendered: Vec> = vec![header]; - let mut any_partial = false; - for r in &report.rows { - if cell_is_partial(&r.coverage.input) - || cell_is_partial(&r.coverage.output) - || cell_is_partial(&r.coverage.reasoning) - || cell_is_partial(&r.coverage.cache_read) - || cell_is_partial(&r.coverage.cache_create) - { - any_partial = true; - } - rendered.push(vec![ - r.label.clone(), - format_uint(r.turns), - coverage_cell(r.usage.input, &r.coverage.input), - coverage_cell(r.usage.output, &r.coverage.output), - coverage_cell(r.usage.reasoning, &r.coverage.reasoning), - coverage_cell(r.usage.cache_read, &r.coverage.cache_read), - coverage_cell( - r.usage.cache_create_5m + r.usage.cache_create_1h, - &r.coverage.cache_create, - ), - format_usd(r.cost.total), - ]); - } - lines.push(render_table(&rendered)); - lines.push(String::new()); - lines.push(format!( - "total cost: {}", - format_usd(report.total_cost.total) - )); - lines.push(format!( - " input {} / output {} / reasoning {} / cacheRead {} / cacheCreate {}", - format_usd(report.total_cost.input), - format_usd(report.total_cost.output), - format_usd(report.total_cost.reasoning), - format_usd(report.total_cost.cache_read), - format_usd(report.total_cost.cache_create), - )); - lines.push(String::new()); - - if report.replacement_savings.calls > 0 { - lines.push(format_replacement_savings_line(&report.replacement_savings)); - lines.push(String::new()); - } - - if !report.stop_reasons.is_empty() { - lines.push(format_stop_reasons_line(&report.stop_reasons)); - lines.push(String::new()); - } - - if !report.subagents.is_empty() { - // `subagents: X paired, Y orphan` — paired sidecars resolved - // via `toolUseResult.agentId`; orphans are the `UnattachedGroup` - // bucket (slash-command synthetic dispatches and crash-mid- - // dispatch sidecars). See AgentWorkforce/burn#435. - lines.push(format_subagents_line(&report.subagents)); - lines.push(String::new()); - } - - if any_partial { - lines.push(format_partial_footer(&report.rows)); - lines.push(String::new()); - } - - if let Some(notice) = render_fidelity_notice(&report.fidelity) { - lines.push(notice); - lines.push(String::new()); - } - - if let Some(q) = report.quality.as_ref() { - lines.push(render_quality(q)); - lines.push(String::new()); - } - - let out = lines.join("\n"); - // TS uses `process.stdout.write(lines.join('\n'))` — no trailing newline. - print!("{}", out); - - if report.unpriced_turns > 0 { - let models = report.unpriced_models.join(", "); - eprintln!( - "warning: {} turn(s) had no pricing for model(s): {} — their cost is reported as $0.", - report.unpriced_turns, models, - ); - eprintln!( - " Update the snapshot (pnpm run pricing:update) or add an override at $RELAYBURN_HOME/models.dev.json.", - ); - } -} - -fn render_quality(q: &QualityResult) -> String { - if q.outcomes.is_empty() { - return "quality: (no sessions)".to_string(); - } - let mut completed = 0u64; - let mut abandoned = 0u64; - let mut errored = 0u64; - let mut unknown = 0u64; - for outcome in &q.outcomes { - match outcome.outcome { - OutcomeLabel::Completed => completed += 1, - OutcomeLabel::Abandoned => abandoned += 1, - OutcomeLabel::Errored => errored += 1, - OutcomeLabel::Unknown => unknown += 1, - } - } - let mut edit_turns = 0u64; - let mut one_shot_turns = 0u64; - for metric in &q.one_shot { - edit_turns += metric.edit_turns; - one_shot_turns += metric.one_shot_turns; - } - let one_shot_line = if edit_turns == 0 { - " one-shot rate: n/a (no edit turns)".to_string() - } else { - format!( - " one-shot rate: {:.1}% across {} edit turns", - (one_shot_turns as f64 / edit_turns as f64) * 100.0, - format_uint(edit_turns), - ) - }; - [ - format!( - "quality — sessions: {}", - format_uint(q.outcomes.len() as u64) - ), - format!( - " outcomes: {} completed / {} abandoned / {} errored / {} unknown", - format_uint(completed), - format_uint(abandoned), - format_uint(errored), - format_uint(unknown), - ), - one_shot_line, - ] - .join("\n") -} - -/// Human-readable outcome line for `burn summary`, e.g. -/// `Turn outcomes: 142 end_turn, 3 max_tokens, 1 refusal, 0 pause`. -/// -/// Always renders `end_turn` / `max_tokens` / `refusal` / `pause` because -/// users want to see the zero — "no refusals" is a meaningful signal. -/// Other buckets (`tool_use`, `stop_sequence`, `silent`, `none`) appear -/// only when non-zero so the line stays scannable. Labels stay snake_case -/// to match the historical Anthropic spelling the issue specified. -fn format_stop_reasons_line(s: &StopReasonCounts) -> String { - let mut parts: Vec = vec![ - format!("{} end_turn", format_uint(s.end_turn)), - format!("{} max_tokens", format_uint(s.max_tokens)), - format!("{} refusal", format_uint(s.refusal)), - format!("{} pause", format_uint(s.pause_turn)), - ]; - if s.tool_use > 0 { - parts.push(format!("{} tool_use", format_uint(s.tool_use))); - } - if s.stop_sequence > 0 { - parts.push(format!("{} stop_sequence", format_uint(s.stop_sequence))); - } - if s.silent > 0 { - parts.push(format!("{} silent", format_uint(s.silent))); - } - if s.none > 0 { - parts.push(format!("{} none", format_uint(s.none))); - } - format!("Turn outcomes: {}", parts.join(", ")) -} - -/// Human-readable subagent line for `burn summary`, e.g. -/// `subagents: 2 paired, 1 orphan`. Both counts are rendered so the line -/// is informative even when one bucket is zero — an orphan-only count -/// flags slash-command synthetic dispatches as a non-trivial signal. -/// See AgentWorkforce/burn#435. -fn format_subagents_line(s: &SubagentCounts) -> String { - format!( - "subagents: {} paired, {} orphan", - format_uint(s.paired), - format_uint(s.orphan), - ) -} - -/// JSON shape for the outcome breakdown. Keys are camelCase to match the -/// rest of the summary surface; every bucket is emitted unconditionally so -/// downstream consumers can index keys without `?` plumbing. -fn stop_reasons_to_json(s: &StopReasonCounts) -> Value { - json!({ - "endTurn": s.end_turn, - "maxTokens": s.max_tokens, - "pauseTurn": s.pause_turn, - "stopSequence": s.stop_sequence, - "toolUse": s.tool_use, - "refusal": s.refusal, - "silent": s.silent, - "none": s.none, - }) -} - -fn format_replacement_savings_line(s: &relayburn_sdk::ReplacementSavingsSummary) -> String { - let call_word = if s.calls == 1 { "call" } else { "calls" }; - format!( - "estimated savings from replacement tools: ~{} tokens across {} {} ({} collapsed vanilla calls)", - format_uint(s.estimated_tokens_saved), - format_uint(s.calls), - call_word, - format_uint(s.collapsed_calls), - ) -} - -/// Footer note explaining the `*` marker. Numerator is the token field with -/// the largest missing count: for each coverage field, sum its `missing` -/// across every row, then take the max. Denominator is the cross-row sum of -/// `known + missing` for `input` (the canonical token field; if a record has -/// any per-turn coverage at all, it carries input). -fn format_partial_footer(rows: &[UsageCostAggregateRow]) -> String { - let mut total: u64 = 0; - for r in rows { - total += r.coverage.input.known + r.coverage.input.missing; - } - let mut missing: u64 = 0; - let mut fields: Vec<&'static str> = Vec::new(); - for f in COVERAGE_FIELDS { - let mut field_missing: u64 = 0; - for r in rows { - field_missing += r.coverage.field(f).missing; - } - match field_missing.cmp(&missing) { - std::cmp::Ordering::Greater => { - missing = field_missing; - fields.clear(); - fields.push(coverage_field_label(f)); - } - std::cmp::Ordering::Equal if field_missing > 0 => { - fields.push(coverage_field_label(f)); - } - _ => {} - } - } - let field = fields.join("/"); - format!( - "{} partial token coverage: largest gap is {} (missing on {} of {} turns); totals still include all turns", - PARTIAL_MARK, - field, - format_uint(missing), - format_uint(total), - ) -} - -fn render_fidelity_notice(f: &FidelitySummary) -> Option { - let usage_only = *f.by_class.get(&FidelityClass::UsageOnly).unwrap_or(&0); - let aggregate_only = *f.by_class.get(&FidelityClass::AggregateOnly).unwrap_or(&0); - let cost_only = *f.by_class.get(&FidelityClass::CostOnly).unwrap_or(&0); - let partial = *f.by_class.get(&FidelityClass::Partial).unwrap_or(&0); - let full = *f.by_class.get(&FidelityClass::Full).unwrap_or(&0); - let non_full = usage_only + aggregate_only + cost_only + partial; - if non_full == 0 && f.unknown == 0 { - return None; - } - let mut parts: Vec = Vec::new(); - if full > 0 { - parts.push(format!("{} full", full)); - } - if usage_only > 0 { - parts.push(format!("{} usage-only", usage_only)); - } - if aggregate_only > 0 { - parts.push(format!("{} aggregate-only", aggregate_only)); - } - if cost_only > 0 { - parts.push(format!("{} cost-only", cost_only)); - } - if partial > 0 { - parts.push(format!("{} partial", partial)); - } - if f.unknown > 0 { - parts.push(format!("{} unknown", f.unknown)); - } - Some(format!( - "fidelity: {} (use --json for per-field coverage)", - parts.join(" / ") - )) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_provider_filter_trims_and_lowercases_csv() { - let got = parse_provider_filter(Some(" Anthropic,OPENAI ,,")) - .unwrap() - .unwrap(); - assert!(got.contains("anthropic")); - assert!(got.contains("openai")); - assert_eq!(got.len(), 2); - assert_eq!( - parse_provider_filter(Some(" , ")), - Err("burn: --provider requires a value"), - ); - } - - #[test] - fn parse_tag_filters_requires_kv_with_non_empty_key() { - let got = parse_tag_filters(&["persona=code-reviewer".to_string()]).unwrap(); - assert_eq!( - got.get("persona").map(String::as_str), - Some("code-reviewer") - ); - - let missing_eq = parse_tag_filters(&["persona".to_string()]).unwrap_err(); - assert!(format!("{missing_eq}").contains("--tag expects k=v")); - - let empty_key = parse_tag_filters(&["=value".to_string()]).unwrap_err(); - assert!(format!("{empty_key}").contains("--tag key must be non-empty")); - - let duplicate = parse_tag_filters(&[ - "persona=code-reviewer".to_string(), - "persona=builder".to_string(), - ]) - .unwrap_err(); - assert!(format!("{duplicate}").contains("duplicate --tag filter")); - } - - #[test] - fn grouped_json_includes_quality_when_report_has_it() { - let report = SummaryGroupedReport { - group_by: SummaryGroupBy::Model, - tag_key: None, - tag_values: Vec::new(), - turn_count: 0, - rows: Vec::new(), - total_cost: CostBreakdown { - model: String::new().into(), - total: 0.0, - input: 0.0, - output: 0.0, - reasoning: 0.0, - cache_read: 0.0, - cache_create: 0.0, - }, - fidelity: relayburn_sdk::summarize_fidelity(&[]), - per_cell_fidelity: json!({"groupBy": "model"}), - replacement_savings: relayburn_sdk::ReplacementSavingsSummary::default(), - stop_reasons: relayburn_sdk::StopReasonCounts::default(), - subagents: SubagentCounts::default(), - quality: Some(QualityResult::default()), - unpriced_turns: 0, - unpriced_models: Vec::new(), - }; - - let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty()); - - assert_eq!(value["quality"], json!({"outcomes": [], "oneShot": []})); - } - - #[test] - fn subagents_line_renders_only_when_counts_nonzero() { - // Empty bucket → skipped, line absent (so old summaries keep - // their byte-identical shape against the existing golden). - let empty = SubagentCounts::default(); - assert!(empty.is_empty()); - - // Non-zero bucket → human line includes both paired+orphan - // counts. Issue #435 explicitly wants both numbers even when - // one is zero, so a slash-command-only session showing only - // orphans is still scannable. - let counts = SubagentCounts { - paired: 2, - orphan: 1, - }; - assert_eq!( - format_subagents_line(&counts), - "subagents: 2 paired, 1 orphan" - ); - } - - #[test] - fn subagents_json_payload_includes_total_and_omits_when_empty() { - // Empty bucket → key absent in JSON so `summary.json | jq` for - // pre-#435 callers still passes without a `?.` guard. - let mut report = SummaryGroupedReport { - group_by: SummaryGroupBy::Model, - tag_key: None, - tag_values: Vec::new(), - turn_count: 0, - rows: Vec::new(), - total_cost: CostBreakdown { - model: String::new().into(), - total: 0.0, - input: 0.0, - output: 0.0, - reasoning: 0.0, - cache_read: 0.0, - cache_create: 0.0, - }, - fidelity: relayburn_sdk::summarize_fidelity(&[]), - per_cell_fidelity: json!({"groupBy": "model"}), - replacement_savings: relayburn_sdk::ReplacementSavingsSummary::default(), - stop_reasons: relayburn_sdk::StopReasonCounts::default(), - subagents: SubagentCounts::default(), - quality: None, - unpriced_turns: 0, - unpriced_models: Vec::new(), - }; - let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty()); - assert!( - value.get("subagents").is_none(), - "subagents key must be omitted when counts are zero; got {value}" - ); - - report.subagents = SubagentCounts { - paired: 2, - orphan: 1, - }; - let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty()); - assert_eq!( - value["subagents"], - json!({"paired": 2, "orphan": 1, "total": 3}), - "non-empty subagent counts must surface `paired`/`orphan`/`total`" - ); - } - - #[test] - fn partial_footer_names_gap_and_says_totals_include_all_turns() { - fn row(label: &str, coverage: relayburn_sdk::RowCoverage) -> UsageCostAggregateRow { - UsageCostAggregateRow { - label: label.to_string(), - turns: 0, - usage: relayburn_sdk::Usage::default(), - cost: CostBreakdown { - model: String::new().into(), - total: 0.0, - input: 0.0, - output: 0.0, - reasoning: 0.0, - cache_read: 0.0, - cache_create: 0.0, - }, - coverage, - } - } - - let mut claude_coverage = relayburn_sdk::RowCoverage::default(); - claude_coverage.input.known = 3; - claude_coverage.output.known = 3; - claude_coverage.reasoning.missing = 3; - - let mut codex_coverage = relayburn_sdk::RowCoverage::default(); - codex_coverage.input.known = 1; - codex_coverage.output.known = 1; - codex_coverage.reasoning.known = 1; - - assert_eq!( - format_partial_footer(&[ - row("claude-sonnet-4-6", claude_coverage), - row("gpt-5-codex", codex_coverage), - ]), - "* partial token coverage: largest gap is reasoning (missing on 3 of 4 turns); totals still include all turns", - ); - } - - #[test] - fn ingest_prelude_text_matches_human_banner() { - let report = relayburn_sdk::IngestReport { - ingested_sessions: 1, - appended_turns: 2_000, - ..relayburn_sdk::IngestReport::empty() - }; - - assert_eq!( - ingest_prelude_text(&report), - "\ningested 1 new session (+2,000 turns)\n", - ); - } -} diff --git a/crates/relayburn-cli/src/commands/summary/human.rs b/crates/relayburn-cli/src/commands/summary/human.rs new file mode 100644 index 00000000..35b13299 --- /dev/null +++ b/crates/relayburn-cli/src/commands/summary/human.rs @@ -0,0 +1,778 @@ +//! Human-readable table rendering and ingest-prelude text for `burn summary`. + +use relayburn_sdk::{ + summary_fidelity_summary_to_value, summary_replacement_savings_to_value, CoverageField, + FidelityClass, FidelitySummary, OutcomeLabel, QualityResult, RelationshipType, + StopReasonCounts, SubagentCounts, SubagentTreeNode, SubagentTypeStats, SummaryByToolReport, + SummaryGroupBy, SummaryGroupedReport, SummaryRelationshipReport, SummarySubagentTreeReport, + UsageCostAggregateRow, +}; +use serde_json::{json, Map, Value}; + +use crate::cli::GlobalArgs; +use crate::render::format::{coerce_whole_f64_to_int, format_uint, format_usd, render_table}; +use crate::render::json::render_json; + +use super::*; + +const COVERAGE_FIELDS: [CoverageField; 5] = [ + CoverageField::Input, + CoverageField::Output, + CoverageField::Reasoning, + CoverageField::CacheRead, + CoverageField::CacheCreate, +]; + +pub(super) fn cell_is_partial(c: &relayburn_sdk::FieldCoverage) -> bool { + c.known > 0 && c.missing > 0 +} + +const PARTIAL_MARK: &str = "*"; +const DASH: &str = "—"; + +pub(super) fn coverage_field_label(field: CoverageField) -> &'static str { + match field { + CoverageField::Input => "input", + CoverageField::Output => "output", + CoverageField::Reasoning => "reasoning", + CoverageField::CacheRead => "cacheRead", + CoverageField::CacheCreate => "cacheCreate", + } +} + +/// Render one token-field cell. Three cases: +/// - every contributing turn reported the field → numeric value, no marker +/// - some turns reported, some didn't → numeric value + `*` +/// - no turn reported → `—` (never `0`, which +/// would falsely claim a real zero from the source) +pub(super) fn coverage_cell(value: u64, c: &relayburn_sdk::FieldCoverage) -> String { + if c.known == 0 && c.missing > 0 { + return DASH.to_string(); + } + if c.known > 0 && c.missing > 0 { + return format!("{}{}", format_uint(value), PARTIAL_MARK); + } + format_uint(value) +} + +pub(super) fn emit_grouped( + globals: &GlobalArgs, + report: &SummaryGroupedReport, + ingest_report: &relayburn_sdk::IngestReport, +) -> std::io::Result<()> { + if globals.json { + return emit_json(report, ingest_report); + } + emit_human(report, ingest_report); + Ok(()) +} + +pub(super) fn emit_ingest_prelude( + globals: &GlobalArgs, + ingest_report: &relayburn_sdk::IngestReport, +) { + if globals.json { + return; + } + emit_human_ingest_prelude(ingest_report); +} + +pub(super) fn emit_human_ingest_prelude(ingest_report: &relayburn_sdk::IngestReport) { + print!("{}", ingest_prelude_text(ingest_report)); +} + +pub(super) fn ingest_prelude_text(ingest_report: &relayburn_sdk::IngestReport) -> String { + format!( + "\ningested {} new session{} (+{} turns)", + ingest_report.ingested_sessions, + if ingest_report.ingested_sessions == 1 { + "" + } else { + "s" + }, + format_uint(ingest_report.appended_turns as u64), + ) + "\n" +} + +pub(super) fn render_by_tool_report( + globals: &GlobalArgs, + report: &SummaryByToolReport, + ingest_report: &relayburn_sdk::IngestReport, +) -> anyhow::Result { + if globals.json { + let by_tool_json: Vec = report + .rows + .iter() + .map(|item| { + let mut row = Map::new(); + row.insert("tool".into(), json!(item.tool)); + row.insert("calls".into(), json!(item.calls)); + row.insert("attributedCost".into(), json!(item.attributed_cost)); + row.insert( + "attributionMethod".into(), + json!(item.attribution_method.wire_str()), + ); + if let Some(s) = item.savings.as_ref() { + row.insert( + "savings".into(), + json!({ + "calls": s.calls, + "collapsedCalls": s.collapsed_calls, + "estimatedTokensSaved": s.estimated_tokens_saved, + }), + ); + } + Value::Object(row) + }) + .collect(); + let mut payload = json!({ + "ingest": { + "ingestedSessions": ingest_report.ingested_sessions, + "appendedTurns": ingest_report.appended_turns, + }, + "turns": report.turn_count, + "byTool": by_tool_json, + "unattributed": report.unattributed_cost, + "fidelity": { "summary": summary_fidelity_summary_to_value(&report.fidelity) }, + }); + if report.replacement_savings.calls > 0 { + payload.as_object_mut().unwrap().insert( + "replacementSavings".into(), + summary_replacement_savings_to_value(&report.replacement_savings), + ); + } + coerce_whole_f64_to_int(&mut payload); + render_json(&payload)?; + return Ok(0); + } + + let mut out = Vec::new(); + out.push(String::new()); + out.push(format!( + "turns analyzed: {}", + format_uint(report.turn_count) + )); + out.push(String::new()); + if report.rows.is_empty() { + out.push("no tool calls found for filters.".to_string()); + let mut text = out.join("\n"); + text.push('\n'); + print!("{text}"); + return Ok(0); + } + + let has_savings = report.replacement_savings.calls > 0; + let mut rows: Vec> = if has_savings { + vec![vec![ + "tool".into(), + "calls".into(), + "attributedCost".into(), + "savedTokens".into(), + ]] + } else { + vec![vec!["tool".into(), "calls".into(), "attributedCost".into()]] + }; + for item in &report.rows { + let mut row = vec![ + item.tool.clone(), + format_uint(item.calls), + format_usd(item.attributed_cost), + ]; + if has_savings { + let saved = item + .savings + .as_ref() + .map(|s| format_uint(s.estimated_tokens_saved)) + .unwrap_or_else(|| "-".to_string()); + row.push(saved); + } + rows.push(row); + } + out.push(render_table(&rows)); + out.push(String::new()); + out.push( + "attributedCost = turn N ingest cost assigned to turn N-1 tool_use blocks by user-turn byte size when available, otherwise split evenly.".to_string(), + ); + out.push(format!( + "unattributed cost (no prior tool call or non-tool user text): {}", + format_usd(report.unattributed_cost), + )); + if has_savings { + out.push(format_replacement_savings_line(&report.replacement_savings)); + } + out.push(String::new()); + print!("{}", out.join("\n")); + Ok(0) +} + +pub(super) fn render_subagent_type_report( + globals: &GlobalArgs, + stats: &[SubagentTypeStats], +) -> anyhow::Result { + if globals.json { + let mut value = serde_json::to_value(stats)?; + coerce_whole_f64_to_int(&mut value); + render_json(&value)?; + return Ok(0); + } + + let mut out = Vec::new(); + out.push(String::new()); + out.push(format!( + "subagent invocations: {}", + format_uint(stats.iter().map(|s| s.invocations).sum()), + )); + out.push(String::new()); + if stats.is_empty() { + out.push(" (no subagent turns in range)".to_string()); + out.push(String::new()); + print!("{}", out.join("\n")); + return Ok(0); + } + out.push(render_subagent_stats_table(stats)); + out.push(String::new()); + print!("{}", out.join("\n")); + Ok(0) +} + +pub(super) fn render_subagent_stats_table(stats: &[SubagentTypeStats]) -> String { + let mut rows = vec![vec![ + "subagentType".into(), + "invocations".into(), + "turns".into(), + "total".into(), + "median".into(), + "p95".into(), + "mean".into(), + ]]; + for s in stats { + rows.push(vec![ + s.subagent_type.clone(), + format_uint(s.invocations), + format_uint(s.turns), + format_usd(s.total_cost), + format_usd(s.median_cost), + format_usd(s.p95_cost), + format_usd(s.mean_cost), + ]); + } + render_table(&rows) +} + +const NO_RELATIONSHIPS_MESSAGE: &str = + "no SessionRelationshipRecord rows found for the matched slice; ingest a session with execution-graph wiring or run `burn state rebuild` once relationship backfill is available"; + +pub(super) fn render_relationship_report( + globals: &GlobalArgs, + report: &SummaryRelationshipReport, +) -> anyhow::Result { + if !report.subagent_types.is_empty() { + return render_relationship_subagent_report(globals, report); + } + if report.relationships.is_empty() { + return render_no_relationships(globals); + } + + if globals.json { + let mut value = json!({ "relationships": report.relationships }); + coerce_whole_f64_to_int(&mut value); + render_json(&value)?; + return Ok(0); + } + + let mut out = Vec::new(); + out.push(String::new()); + out.push(format!( + "relationships: {}", + format_uint(report.relationships.iter().map(|s| s.count).sum()), + )); + out.push(String::new()); + let mut rows = vec![vec![ + "relationshipType".into(), + "count".into(), + "turnCount".into(), + "total".into(), + "median".into(), + "p95".into(), + "mean".into(), + ]]; + for s in &report.relationships { + rows.push(vec![ + s.relationship_type.wire_str().to_string(), + format_uint(s.count), + format_uint(s.turn_count), + format_usd(s.total_cost), + format_usd(s.median_cost), + format_usd(s.p95_cost), + format_usd(s.mean_cost), + ]); + } + out.push(render_table(&rows)); + out.push(String::new()); + print!("{}", out.join("\n")); + Ok(0) +} + +pub(super) fn render_relationship_subagent_report( + globals: &GlobalArgs, + report: &SummaryRelationshipReport, +) -> anyhow::Result { + if report.subagent_types.is_empty() { + if globals.json { + let mut value = json!({ + "relationships": [], + "subagentTypes": [], + "message": NO_RELATIONSHIPS_MESSAGE, + }); + coerce_whole_f64_to_int(&mut value); + render_json(&value)?; + return Ok(0); + } + return render_no_relationships(globals); + } + if globals.json { + let mut value = json!({ + "relationships": report.relationships, + "subagentTypes": report.subagent_types, + }); + coerce_whole_f64_to_int(&mut value); + render_json(&value)?; + return Ok(0); + } + + let mut out = Vec::new(); + out.push(String::new()); + out.push(format!( + "subagent invocations: {}", + format_uint(report.subagent_types.iter().map(|s| s.invocations).sum()), + )); + out.push(String::new()); + let mut rows = vec![vec![ + "subagentType".into(), + "invocations".into(), + "turns".into(), + "total".into(), + "median".into(), + "p95".into(), + "mean".into(), + ]]; + for s in &report.subagent_types { + rows.push(vec![ + s.subagent_type.clone(), + format_uint(s.invocations), + format_uint(s.turns), + format_usd(s.total_cost), + format_usd(s.median_cost), + format_usd(s.p95_cost), + format_usd(s.mean_cost), + ]); + } + out.push(render_table(&rows)); + out.push(String::new()); + print!("{}", out.join("\n")); + Ok(0) +} + +pub(super) fn render_no_relationships(globals: &GlobalArgs) -> anyhow::Result { + if globals.json { + render_json(&json!({ + "relationships": [], + "message": NO_RELATIONSHIPS_MESSAGE, + }))?; + } else { + println!("{NO_RELATIONSHIPS_MESSAGE}"); + } + Ok(0) +} + +pub(super) fn render_subagent_tree_report( + globals: &GlobalArgs, + report: &SummarySubagentTreeReport, +) -> anyhow::Result { + if globals.json { + let root = match report.root.as_ref() { + Some(root) => serde_json::to_value(root)?, + None => Value::Null, + }; + let mut value = json!({ + "sessionId": report.session_id.as_str(), + "root": root, + }); + coerce_whole_f64_to_int(&mut value); + render_json(&value)?; + return Ok(0); + } + + let Some(root) = report.root.as_ref() else { + println!("no turns found for session {}", report.session_id); + return Ok(0); + }; + + let mut out = Vec::new(); + out.push(String::new()); + out.push(format!("session: {}", report.session_id)); + out.push(format!( + "total: {} across {} turn{}", + format_usd(root.cumulative_cost), + format_uint(root.cumulative_turns), + if root.cumulative_turns == 1 { "" } else { "s" }, + )); + out.push(String::new()); + out.extend(render_tree(root)); + out.push(String::new()); + print!("{}", out.join("\n")); + Ok(0) +} + +pub(super) fn render_tree(root: &SubagentTreeNode) -> Vec { + let mut out = Vec::new(); + out.push(render_node_line(root, "")); + render_children(root, "", &mut out); + out +} + +pub(super) fn render_children(node: &SubagentTreeNode, prefix: &str, out: &mut Vec) { + let n = node.children.len(); + for (i, child) in node.children.iter().enumerate() { + let is_last = i == n - 1; + let branch = if is_last { "└─ " } else { "├─ " }; + out.push(render_node_line(child, &format!("{prefix}{branch}"))); + let child_prefix = if is_last { + format!("{prefix} ") + } else { + format!("{prefix}│ ") + }; + render_children(child, &child_prefix, out); + } +} + +pub(super) fn render_node_line(node: &SubagentTreeNode, indent: &str) -> String { + let relationship = if node.relationship_type != RelationshipType::Root + && node.relationship_type != RelationshipType::Subagent + { + format!(" [{}]", node.relationship_type.wire_str()) + } else { + String::new() + }; + let model = if node.models.is_empty() { + String::new() + } else { + format!(" ({})", node.models.join(", ")) + }; + format!( + "{}{}{}{} {} [{} turn{}]", + indent, + node.label, + relationship, + model, + format_usd(node.cumulative_cost), + format_uint(node.cumulative_turns), + if node.cumulative_turns == 1 { "" } else { "s" }, + ) +} + +pub(super) fn emit_human( + report: &SummaryGroupedReport, + ingest_report: &relayburn_sdk::IngestReport, +) { + let mut lines: Vec = Vec::new(); + emit_human_ingest_prelude(ingest_report); + lines.push(String::new()); + + lines.push(format!( + "turns analyzed: {}", + format_uint(report.turn_count) + )); + lines.push(String::new()); + + if report.rows.is_empty() { + lines.push("no turns match the current filters.".to_string()); + let mut out = lines.join("\n"); + out.push('\n'); + print!("{}", out); + return; + } + + let header_label = if report.group_by == SummaryGroupBy::Tag { + "value" + } else { + report.group_by.wire_str() + }; + let header = vec![ + header_label.to_string(), + "turns".into(), + "input".into(), + "output".into(), + "reasoning".into(), + "cacheRead".into(), + "cacheCreate".into(), + "cost".into(), + ]; + let mut rendered: Vec> = vec![header]; + let mut any_partial = false; + for r in &report.rows { + if cell_is_partial(&r.coverage.input) + || cell_is_partial(&r.coverage.output) + || cell_is_partial(&r.coverage.reasoning) + || cell_is_partial(&r.coverage.cache_read) + || cell_is_partial(&r.coverage.cache_create) + { + any_partial = true; + } + rendered.push(vec![ + r.label.clone(), + format_uint(r.turns), + coverage_cell(r.usage.input, &r.coverage.input), + coverage_cell(r.usage.output, &r.coverage.output), + coverage_cell(r.usage.reasoning, &r.coverage.reasoning), + coverage_cell(r.usage.cache_read, &r.coverage.cache_read), + coverage_cell( + r.usage.cache_create_5m + r.usage.cache_create_1h, + &r.coverage.cache_create, + ), + format_usd(r.cost.total), + ]); + } + lines.push(render_table(&rendered)); + lines.push(String::new()); + lines.push(format!( + "total cost: {}", + format_usd(report.total_cost.total) + )); + lines.push(format!( + " input {} / output {} / reasoning {} / cacheRead {} / cacheCreate {}", + format_usd(report.total_cost.input), + format_usd(report.total_cost.output), + format_usd(report.total_cost.reasoning), + format_usd(report.total_cost.cache_read), + format_usd(report.total_cost.cache_create), + )); + lines.push(String::new()); + + if report.replacement_savings.calls > 0 { + lines.push(format_replacement_savings_line(&report.replacement_savings)); + lines.push(String::new()); + } + + if !report.stop_reasons.is_empty() { + lines.push(format_stop_reasons_line(&report.stop_reasons)); + lines.push(String::new()); + } + + if !report.subagents.is_empty() { + // `subagents: X paired, Y orphan` — paired sidecars resolved + // via `toolUseResult.agentId`; orphans are the `UnattachedGroup` + // bucket (slash-command synthetic dispatches and crash-mid- + // dispatch sidecars). See AgentWorkforce/burn#435. + lines.push(format_subagents_line(&report.subagents)); + lines.push(String::new()); + } + + if any_partial { + lines.push(format_partial_footer(&report.rows)); + lines.push(String::new()); + } + + if let Some(notice) = render_fidelity_notice(&report.fidelity) { + lines.push(notice); + lines.push(String::new()); + } + + if let Some(q) = report.quality.as_ref() { + lines.push(render_quality(q)); + lines.push(String::new()); + } + + let out = lines.join("\n"); + // TS uses `process.stdout.write(lines.join('\n'))` — no trailing newline. + print!("{}", out); + + if report.unpriced_turns > 0 { + let models = report.unpriced_models.join(", "); + eprintln!( + "warning: {} turn(s) had no pricing for model(s): {} — their cost is reported as $0.", + report.unpriced_turns, models, + ); + eprintln!( + " Update the snapshot (pnpm run pricing:update) or add an override at $RELAYBURN_HOME/models.dev.json.", + ); + } +} + +pub(super) fn render_quality(q: &QualityResult) -> String { + if q.outcomes.is_empty() { + return "quality: (no sessions)".to_string(); + } + let mut completed = 0u64; + let mut abandoned = 0u64; + let mut errored = 0u64; + let mut unknown = 0u64; + for outcome in &q.outcomes { + match outcome.outcome { + OutcomeLabel::Completed => completed += 1, + OutcomeLabel::Abandoned => abandoned += 1, + OutcomeLabel::Errored => errored += 1, + OutcomeLabel::Unknown => unknown += 1, + } + } + let mut edit_turns = 0u64; + let mut one_shot_turns = 0u64; + for metric in &q.one_shot { + edit_turns += metric.edit_turns; + one_shot_turns += metric.one_shot_turns; + } + let one_shot_line = if edit_turns == 0 { + " one-shot rate: n/a (no edit turns)".to_string() + } else { + format!( + " one-shot rate: {:.1}% across {} edit turns", + (one_shot_turns as f64 / edit_turns as f64) * 100.0, + format_uint(edit_turns), + ) + }; + [ + format!( + "quality — sessions: {}", + format_uint(q.outcomes.len() as u64) + ), + format!( + " outcomes: {} completed / {} abandoned / {} errored / {} unknown", + format_uint(completed), + format_uint(abandoned), + format_uint(errored), + format_uint(unknown), + ), + one_shot_line, + ] + .join("\n") +} + +/// Human-readable outcome line for `burn summary`, e.g. +/// `Turn outcomes: 142 end_turn, 3 max_tokens, 1 refusal, 0 pause`. +/// +/// Always renders `end_turn` / `max_tokens` / `refusal` / `pause` because +/// users want to see the zero — "no refusals" is a meaningful signal. +/// Other buckets (`tool_use`, `stop_sequence`, `silent`, `none`) appear +/// only when non-zero so the line stays scannable. Labels stay snake_case +/// to match the historical Anthropic spelling the issue specified. +pub(super) fn format_stop_reasons_line(s: &StopReasonCounts) -> String { + let mut parts: Vec = vec![ + format!("{} end_turn", format_uint(s.end_turn)), + format!("{} max_tokens", format_uint(s.max_tokens)), + format!("{} refusal", format_uint(s.refusal)), + format!("{} pause", format_uint(s.pause_turn)), + ]; + if s.tool_use > 0 { + parts.push(format!("{} tool_use", format_uint(s.tool_use))); + } + if s.stop_sequence > 0 { + parts.push(format!("{} stop_sequence", format_uint(s.stop_sequence))); + } + if s.silent > 0 { + parts.push(format!("{} silent", format_uint(s.silent))); + } + if s.none > 0 { + parts.push(format!("{} none", format_uint(s.none))); + } + format!("Turn outcomes: {}", parts.join(", ")) +} + +/// Human-readable subagent line for `burn summary`, e.g. +/// `subagents: 2 paired, 1 orphan`. Both counts are rendered so the line +/// is informative even when one bucket is zero — an orphan-only count +/// flags slash-command synthetic dispatches as a non-trivial signal. +/// See AgentWorkforce/burn#435. +pub(super) fn format_subagents_line(s: &SubagentCounts) -> String { + format!( + "subagents: {} paired, {} orphan", + format_uint(s.paired), + format_uint(s.orphan), + ) +} + +pub(super) fn format_replacement_savings_line( + s: &relayburn_sdk::ReplacementSavingsSummary, +) -> String { + let call_word = if s.calls == 1 { "call" } else { "calls" }; + format!( + "estimated savings from replacement tools: ~{} tokens across {} {} ({} collapsed vanilla calls)", + format_uint(s.estimated_tokens_saved), + format_uint(s.calls), + call_word, + format_uint(s.collapsed_calls), + ) +} + +/// Footer note explaining the `*` marker. Numerator is the token field with +/// the largest missing count: for each coverage field, sum its `missing` +/// across every row, then take the max. Denominator is the cross-row sum of +/// `known + missing` for `input` (the canonical token field; if a record has +/// any per-turn coverage at all, it carries input). +pub(super) fn format_partial_footer(rows: &[UsageCostAggregateRow]) -> String { + let mut total: u64 = 0; + for r in rows { + total += r.coverage.input.known + r.coverage.input.missing; + } + let mut missing: u64 = 0; + let mut fields: Vec<&'static str> = Vec::new(); + for f in COVERAGE_FIELDS { + let mut field_missing: u64 = 0; + for r in rows { + field_missing += r.coverage.field(f).missing; + } + match field_missing.cmp(&missing) { + std::cmp::Ordering::Greater => { + missing = field_missing; + fields.clear(); + fields.push(coverage_field_label(f)); + } + std::cmp::Ordering::Equal if field_missing > 0 => { + fields.push(coverage_field_label(f)); + } + _ => {} + } + } + let field = fields.join("/"); + format!( + "{} partial token coverage: largest gap is {} (missing on {} of {} turns); totals still include all turns", + PARTIAL_MARK, + field, + format_uint(missing), + format_uint(total), + ) +} + +pub(super) fn render_fidelity_notice(f: &FidelitySummary) -> Option { + let usage_only = *f.by_class.get(&FidelityClass::UsageOnly).unwrap_or(&0); + let aggregate_only = *f.by_class.get(&FidelityClass::AggregateOnly).unwrap_or(&0); + let cost_only = *f.by_class.get(&FidelityClass::CostOnly).unwrap_or(&0); + let partial = *f.by_class.get(&FidelityClass::Partial).unwrap_or(&0); + let full = *f.by_class.get(&FidelityClass::Full).unwrap_or(&0); + let non_full = usage_only + aggregate_only + cost_only + partial; + if non_full == 0 && f.unknown == 0 { + return None; + } + let mut parts: Vec = Vec::new(); + if full > 0 { + parts.push(format!("{} full", full)); + } + if usage_only > 0 { + parts.push(format!("{} usage-only", usage_only)); + } + if aggregate_only > 0 { + parts.push(format!("{} aggregate-only", aggregate_only)); + } + if cost_only > 0 { + parts.push(format!("{} cost-only", cost_only)); + } + if partial > 0 { + parts.push(format!("{} partial", partial)); + } + if f.unknown > 0 { + parts.push(format!("{} unknown", f.unknown)); + } + Some(format!( + "fidelity: {} (use --json for per-field coverage)", + parts.join(" / ") + )) +} diff --git a/crates/relayburn-cli/src/commands/summary/json.rs b/crates/relayburn-cli/src/commands/summary/json.rs new file mode 100644 index 00000000..6e448833 --- /dev/null +++ b/crates/relayburn-cli/src/commands/summary/json.rs @@ -0,0 +1,170 @@ +//! JSON serialization for `burn summary` reports. + +use relayburn_sdk::{ + summary_fidelity_summary_to_value, summary_replacement_savings_to_value, CostBreakdown, + StopReasonCounts, SummaryGroupBy, SummaryGroupedReport, +}; +use serde_json::{json, Map, Value}; + +use crate::cli::GlobalArgs; +use crate::render::format::{coerce_whole_f64_to_int, format_uint, format_usd}; +use crate::render::json::render_json; + +use super::*; + +pub(super) fn emit_json( + report: &SummaryGroupedReport, + ingest_report: &relayburn_sdk::IngestReport, +) -> std::io::Result<()> { + let value = grouped_json_value(report, ingest_report); + render_json(&value) +} + +/// Render a `--bucket` time-series. JSON emits `{ bucketSeconds, buckets: [...] }` +/// (the consumer); human output is one line per bucket. +pub(super) fn emit_summary_timeseries( + globals: &GlobalArgs, + series: &relayburn_sdk::SummaryTimeseries, + ingest_report: &relayburn_sdk::IngestReport, +) -> anyhow::Result { + if globals.json { + render_json(series)?; + return Ok(0); + } + emit_human_ingest_prelude(ingest_report); + if series.buckets.is_empty() { + println!("(no data in range)"); + return Ok(0); + } + for bucket in &series.buckets { + println!( + "{} {:>5} turns {:>14} tok {}", + bucket.start, + bucket.turn_count, + format_uint(bucket.total_tokens), + format_usd(bucket.total_cost.total), + ); + } + Ok(0) +} + +pub(super) fn grouped_json_value( + report: &SummaryGroupedReport, + ingest_report: &relayburn_sdk::IngestReport, +) -> Value { + let key = report.group_by.json_key(); + let label_key = report.group_by.wire_str(); + + let group_rows: Vec = report + .rows + .iter() + .enumerate() + .map(|(idx, r)| { + let mut row = if report.group_by == SummaryGroupBy::Tag { + json!({ + "tag": report.tag_key.as_deref().unwrap_or(""), + "value": report.tag_values.get(idx).cloned().flatten(), + }) + } else { + json!({ + label_key: r.label, + }) + }; + let obj = row.as_object_mut().unwrap(); + obj.insert("turns".into(), json!(r.turns)); + obj.insert( + "usage".into(), + json!({ + "input": r.usage.input, + "output": r.usage.output, + "reasoning": r.usage.reasoning, + "cacheRead": r.usage.cache_read, + "cacheCreate5m": r.usage.cache_create_5m, + "cacheCreate1h": r.usage.cache_create_1h, + }), + ); + obj.insert("cost".into(), cost_breakdown_to_json(&r.cost)); + row + }) + .collect(); + + let mut payload = Map::new(); + payload.insert( + "ingest".into(), + json!({ + "ingestedSessions": ingest_report.ingested_sessions, + "appendedTurns": ingest_report.appended_turns, + }), + ); + payload.insert("turns".into(), json!(report.turn_count)); + payload.insert( + "totalCost".into(), + cost_breakdown_to_json(&report.total_cost), + ); + payload.insert(key.into(), Value::Array(group_rows)); + payload.insert( + "fidelity".into(), + json!({ + "summary": summary_fidelity_summary_to_value(&report.fidelity), + "perCell": report.per_cell_fidelity.clone(), + }), + ); + if report.replacement_savings.calls > 0 { + payload.insert( + "replacementSavings".into(), + summary_replacement_savings_to_value(&report.replacement_savings), + ); + } + payload.insert( + "stopReasons".into(), + stop_reasons_to_json(&report.stop_reasons), + ); + if !report.subagents.is_empty() { + // `subagents: {paired, orphan, total}` (issue #435). Skipped + // when both buckets are zero so the JSON shape stays compact + // for sessions that never spawned a subagent. + payload.insert( + "subagents".into(), + json!({ + "paired": report.subagents.paired, + "orphan": report.subagents.orphan, + "total": report.subagents.total(), + }), + ); + } + if let Some(quality) = report.quality.as_ref() { + payload.insert("quality".into(), json!(quality)); + } + + let mut value = Value::Object(payload); + coerce_whole_f64_to_int(&mut value); + value +} + +pub(super) fn cost_breakdown_to_json(c: &CostBreakdown) -> Value { + json!({ + "model": c.model.as_ref(), + "total": c.total, + "input": c.input, + "output": c.output, + "reasoning": c.reasoning, + "cacheRead": c.cache_read, + "cacheCreate": c.cache_create, + }) +} + +/// JSON shape for the outcome breakdown. Keys are camelCase to match the +/// rest of the summary surface; every bucket is emitted unconditionally so +/// downstream consumers can index keys without `?` plumbing. +pub(super) fn stop_reasons_to_json(s: &StopReasonCounts) -> Value { + json!({ + "endTurn": s.end_turn, + "maxTokens": s.max_tokens, + "pauseTurn": s.pause_turn, + "stopSequence": s.stop_sequence, + "toolUse": s.tool_use, + "refusal": s.refusal, + "silent": s.silent, + "none": s.none, + }) +} diff --git a/crates/relayburn-cli/src/commands/summary/mod.rs b/crates/relayburn-cli/src/commands/summary/mod.rs new file mode 100644 index 00000000..28c5babf --- /dev/null +++ b/crates/relayburn-cli/src/commands/summary/mod.rs @@ -0,0 +1,611 @@ +//! `burn summary` — aggregate session usage and cost. +//! +//! Thin presenter over the `relayburn_sdk` query helpers. Mirrors the TS +//! `packages/cli/src/commands/summary.ts` surface: grouped model/provider +//! summaries, tool attribution, subagent views, relationship rollups, workflow +//! / agent / provider filters, and the optional quality footer. +//! +//! ## Wiring +//! +//! 1. Open a [`relayburn_sdk::LedgerHandle`] honoring `--ledger-path` / +//! `RELAYBURN_HOME`. +//! 2. Optionally run [`relayburn_sdk::ingest_all`] against the same handle, +//! gated on `--ingest`. `summary` is a read verb, and a pre-query sweep +//! re-stats every session file under every harness store — seconds on a +//! large OpenCode store (tens of thousands of sessions) — so it is +//! **off by default**. The steady-state setup keeps the ledger fresh out +//! of band: `burn ingest --watch` once per host, or the Claude Stop hook +//! (`burn ingest --hook claude`) firing each turn. Pass `--ingest` for a +//! one-off freshen; the human banner then leads with +//! `ingested N new sessions (+M turns)` and a TTY gets a stderr spinner. +//! When the sweep is skipped, an empty [`relayburn_sdk::IngestReport`] +//! keeps the banner / JSON `ingest` block byte-identical to a no-op sweep +//! (`ingested 0 new sessions`). See `burn compare` for the same decision. +//! 3. Lower CLI flags into [`relayburn_sdk::SummaryReportOptions`] and call +//! the SDK-owned `summary_report` verb. +//! 4. Render the typed report as JSON or human output. + +use std::collections::{BTreeMap, BTreeSet}; + +use clap::Args; +use relayburn_sdk::{ + ingest_all, Enrichment, Ledger, LedgerHandle, LedgerOpenOptions, SummaryReport, + SummaryReportMode, SummaryReportOptions, +}; + +use crate::cli::GlobalArgs; +use crate::render::error::report_error; +use crate::render::progress::TaskProgress; + +mod human; +mod json; + +use human::*; +use json::*; + +#[cfg(test)] +use relayburn_sdk::{ + CostBreakdown, QualityResult, SubagentCounts, SummaryGroupBy, SummaryGroupedReport, + UsageCostAggregateRow, +}; +#[cfg(test)] +use serde_json::json; + +/// Per-command flags for `burn summary`. Mirrors the TS surface in +/// `packages/cli/src/commands/summary.ts` so a TS user can carry their +/// muscle memory across. +#[derive(Debug, Clone, Args)] +pub struct SummaryArgs { + /// Slice the ledger to events at or after ``. Accepts either an + /// ISO timestamp or a relative range (`24h`, `7d`, `4w`, `2m`). + #[arg(long, value_name = "WHEN")] + pub since: Option, + + /// Restrict to a single project (matches `project` or `projectKey`). + #[arg(long, value_name = "PROJECT")] + pub project: Option, + + /// Restrict to a single session id. + #[arg(long, value_name = "SESSION_ID")] + pub session: Option, + + /// Group by effective provider instead of model. + #[arg(long = "by-provider")] + pub by_provider: bool, + + /// Group by tool, attributing each turn's ingest cost to the prior + /// turn's `tool_use` blocks. Emits a `byTool` table; mutually + /// exclusive with `--by-provider` / the subagent flags. + #[arg(long = "by-tool")] + pub by_tool: bool, + + /// Bucket by `subagent.subagentType`. Mutually exclusive with the + /// other group-by flags. + #[arg(long = "by-subagent-type")] + pub by_subagent_type: bool, + + /// Bucket by `SessionRelationshipRecord.relationshipType` (or pass + /// `subagent` to drill into the subagent leaf). Mutually exclusive + /// with `--subagent-tree`. + #[arg(long = "by-relationship", value_name = "MODE", num_args = 0..=1, default_missing_value = "")] + pub by_relationship: Option, + + /// Render the subagent spawn tree for a session id. Passing the flag + /// without a value uses `--session`. + #[arg(long = "subagent-tree", value_name = "SESSION_ID", num_args = 0..=1, default_missing_value = "")] + pub subagent_tree: Option, + + /// Restrict the subagent tree / relationship views to a single agent + /// id. + #[arg(long, value_name = "AGENT_ID")] + pub agent: Option, + + /// Filter by enrichment workflow id. + #[arg(long, value_name = "WORKFLOW_ID")] + pub workflow: Option, + + /// Filter by folded enrichment tag. Repeatable; every tag must match. + #[arg(long = "tag", value_name = "K=V")] + pub tag: Vec, + + /// Group totals by a folded enrichment tag value. + #[arg(long = "group-by-tag", value_name = "KEY")] + pub group_by_tag: Option, + + /// Provider filter (CSV of provider names; case-insensitive). + #[arg(long, value_name = "PROVIDERS")] + pub provider: Option, + + /// Append a quality summary (one-shot rate, completion outcomes). + #[arg(long)] + pub quality: bool, + + /// Accepted for TS CLI flag parity; a no-op against the Rust SDK, + /// which is SQLite-native and has no archive layer to bypass. + #[arg(long = "no-archive")] + pub no_archive: bool, + + /// Run a pre-query ingest sweep so the summary leads with freshly + /// appended sessions. Off by default: `summary` is a read verb, and a + /// full-store sweep re-stats every session file under every harness + /// store — seconds on a large ledger. Keep the ledger current out of + /// band with `burn ingest --watch` (or the Claude Stop hook); pass + /// `--ingest` only for a one-off freshen. + #[arg(long = "ingest")] + pub ingest: bool, + + /// Emit a time-series instead of a single total: bucket the `--since` + /// window into fixed-width windows and report per-bucket cost/usage. + /// Duration grammar: `30s`, `5m` (minutes), `1h`, `12h`, `1d`, `7d`. + /// Only valid for the default grouped (`byModel`/`--by-provider`) summary. + #[arg(long, value_name = "DURATION")] + pub bucket: Option, +} + +pub fn run(globals: &GlobalArgs, args: SummaryArgs) -> i32 { + match run_inner(globals, args) { + Ok(code) => code, + Err(err) => report_error(&err, globals), + } +} + +fn run_inner(globals: &GlobalArgs, args: SummaryArgs) -> anyhow::Result { + // Mode exclusivity — mirror the TS CLI's stderr+exit2 contract so a + // mis-typed combination of flags produces a clear message rather than + // silently dropping one. + if args.by_tool + && (args.by_provider + || args.by_subagent_type + || args.by_relationship.is_some() + || args.subagent_tree.is_some() + || args.group_by_tag.is_some()) + { + eprintln!( + "burn: --by-tool cannot be combined with --by-provider/--by-subagent-type/--by-relationship/--subagent-tree/--group-by-tag" + ); + return Ok(2); + } + if args.by_provider + && (args.by_subagent_type + || args.by_relationship.is_some() + || args.subagent_tree.is_some() + || args.group_by_tag.is_some()) + { + eprintln!( + "burn: --by-provider cannot be combined with --by-subagent-type/--by-relationship/--subagent-tree/--group-by-tag" + ); + return Ok(2); + } + if args.by_subagent_type + && (args.by_relationship.is_some() + || args.subagent_tree.is_some() + || args.group_by_tag.is_some()) + { + eprintln!( + "burn: --by-subagent-type cannot be combined with --by-relationship/--subagent-tree/--group-by-tag" + ); + return Ok(2); + } + if args.by_relationship.is_some() + && (args.subagent_tree.is_some() || args.group_by_tag.is_some()) + { + eprintln!("burn: --by-relationship cannot be combined with --subagent-tree/--group-by-tag"); + return Ok(2); + } + if args.subagent_tree.is_some() && args.group_by_tag.is_some() { + eprintln!("burn: --subagent-tree cannot be combined with --group-by-tag"); + return Ok(2); + } + if let Some(rel) = &args.by_relationship { + if !rel.is_empty() && rel != "subagent" { + eprintln!("burn: --by-relationship accepts only the optional value \"subagent\""); + return Ok(2); + } + } + if let Some(tag_key) = args.group_by_tag.as_deref() { + if tag_key.is_empty() { + eprintln!("burn: --group-by-tag requires a non-empty key"); + return Ok(2); + } + } + + // `--bucket` opts into a per-bucket time-series. Parse and validate it + // (including the mode/flag combinations it supports) before opening the + // ledger or running ingest, so a bad invocation fails fast. + let bucket_secs = if let Some(bucket_raw) = args.bucket.as_deref() { + if args.by_tool + || args.by_subagent_type + || args.by_relationship.is_some() + || args.subagent_tree.is_some() + || args.group_by_tag.is_some() + { + eprintln!( + "burn: --bucket is only supported with the default grouped summary or --by-provider" + ); + return Ok(2); + } + if args.quality { + eprintln!("burn: --bucket is not supported with --quality"); + return Ok(2); + } + match relayburn_sdk::parse_bucket(bucket_raw) { + Ok(secs) => Some(secs), + Err(err) => { + eprintln!("burn: {err}"); + return Ok(2); + } + } + } else { + None + }; + + let provider_filter = match parse_provider_filter(args.provider.as_deref()) { + Ok(filter) => filter, + Err(msg) => { + eprintln!("{msg}"); + return Ok(2); + } + }; + let tag_filter: Enrichment = match parse_tag_filters(&args.tag) { + Ok(filter) => filter, + Err(err) => { + eprintln!("{err}"); + return Ok(2); + } + }; + let subagent_tree_session_id = if let Some(tree_flag) = args.subagent_tree.as_deref() { + if tree_flag.is_empty() && args.session.is_none() { + eprintln!("burn: --subagent-tree requires a session id (positional or --session)"); + return Ok(2); + } + Some(if tree_flag.is_empty() { + None + } else { + Some(tree_flag.to_string()) + }) + } else { + None + }; + + // `--no-archive` is accepted for TS CLI flag parity but is a no-op: + // the Rust SDK is SQLite-native and has no archive layer to bypass. + let _ = args.no_archive; + let progress = TaskProgress::new(globals, "summary"); + + let opts = match globals.ledger_path.as_deref() { + Some(h) => LedgerOpenOptions::with_home(h), + None => LedgerOpenOptions::default(), + }; + progress.set_task("opening ledger"); + let mut handle = Ledger::open(opts).inspect_err(|_| { + progress.finish_and_clear(); + })?; + + // Read-verb default: skip the pre-query sweep (see the module doc and + // `burn compare`). `--ingest` opts back into a one-off freshen; otherwise + // an empty report keeps the banner / JSON `ingest` block identical to a + // no-op sweep without paying for the full-store stat walk. + let ingest_report = if args.ingest { + run_ingest(&mut handle, &progress, globals.ledger_path.clone()).inspect_err(|_| { + progress.finish_and_clear(); + })? + } else { + relayburn_sdk::IngestReport::empty() + }; + + let mode = if let Some(session_id) = subagent_tree_session_id { + SummaryReportMode::SubagentTree { session_id } + } else if args.by_tool { + SummaryReportMode::ByTool + } else if args.by_subagent_type { + SummaryReportMode::BySubagentType + } else if let Some(rel_flag) = args.by_relationship.as_deref() { + SummaryReportMode::ByRelationship { + subagent: rel_flag == "subagent", + } + } else { + SummaryReportMode::Grouped { + by_provider: args.by_provider, + } + }; + + let opts = SummaryReportOptions { + session: args.session, + project: args.project, + since: args.since, + workflow: args.workflow, + tags: if tag_filter.is_empty() { + None + } else { + Some(tag_filter) + }, + group_by_tag: args.group_by_tag, + agent: args.agent, + providers: provider_filter.map(|providers| providers.into_iter().collect()), + mode, + include_quality: args.quality, + ledger_home: None, + }; + + // `--bucket` switches to a per-bucket time-series of the grouped summary. + // Parsing/validation already happened above, before the ledger was opened. + if let Some(bucket_secs) = bucket_secs { + progress.set_task("building summary time-series"); + let series = handle + .summary_timeseries(opts, bucket_secs) + .inspect_err(|_| { + progress.finish_and_clear(); + })?; + progress.finish_and_clear(); + return emit_summary_timeseries(globals, &series, &ingest_report); + } + + progress.set_task("building summary"); + let report = handle.summary_report(opts).inspect_err(|_| { + progress.finish_and_clear(); + })?; + progress.finish_and_clear(); + + match report { + SummaryReport::Grouped(report) => { + emit_grouped(globals, &report, &ingest_report)?; + } + SummaryReport::ByTool(report) => { + emit_ingest_prelude(globals, &ingest_report); + return render_by_tool_report(globals, &report, &ingest_report); + } + SummaryReport::BySubagentType(report) => { + emit_ingest_prelude(globals, &ingest_report); + return render_subagent_type_report(globals, &report.stats); + } + SummaryReport::Relationship(report) => { + emit_ingest_prelude(globals, &ingest_report); + return render_relationship_report(globals, &report); + } + SummaryReport::SubagentTree(report) => { + emit_ingest_prelude(globals, &ingest_report); + return render_subagent_tree_report(globals, &report); + } + } + Ok(0) +} + +fn parse_provider_filter(raw: Option<&str>) -> Result>, &'static str> { + let Some(raw) = raw else { + return Ok(None); + }; + let providers: BTreeSet = raw + .split(',') + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty()) + .collect(); + if providers.is_empty() { + return Err("burn: --provider requires a value"); + } + Ok(Some(providers)) +} + +fn parse_tag_filters(tags: &[String]) -> anyhow::Result> { + let mut out = BTreeMap::new(); + for raw in tags { + let (key, value) = raw + .split_once('=') + .ok_or_else(|| anyhow::anyhow!("burn: --tag expects k=v, got \"{raw}\""))?; + if key.is_empty() { + anyhow::bail!("burn: --tag key must be non-empty (got \"{raw}\")"); + } + if let Some(existing) = out.get(key) { + anyhow::bail!( + "burn: duplicate --tag filter for key \"{key}\" ({existing:?} vs {value:?})" + ); + } + out.insert(key.to_string(), value.to_string()); + } + Ok(out) +} + +/// Run an ingest sweep on the open handle. +fn run_ingest( + handle: &mut LedgerHandle, + progress: &TaskProgress, + ledger_home: Option, +) -> anyhow::Result { + progress.set_task("refreshing ledger"); + let opts = progress.ingest_options(ledger_home); + ingest_all(handle.raw_mut(), &opts).inspect_err(|_| { + progress.finish_and_clear(); + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_provider_filter_trims_and_lowercases_csv() { + let got = parse_provider_filter(Some(" Anthropic,OPENAI ,,")) + .unwrap() + .unwrap(); + assert!(got.contains("anthropic")); + assert!(got.contains("openai")); + assert_eq!(got.len(), 2); + assert_eq!( + parse_provider_filter(Some(" , ")), + Err("burn: --provider requires a value"), + ); + } + + #[test] + fn parse_tag_filters_requires_kv_with_non_empty_key() { + let got = parse_tag_filters(&["persona=code-reviewer".to_string()]).unwrap(); + assert_eq!( + got.get("persona").map(String::as_str), + Some("code-reviewer") + ); + + let missing_eq = parse_tag_filters(&["persona".to_string()]).unwrap_err(); + assert!(format!("{missing_eq}").contains("--tag expects k=v")); + + let empty_key = parse_tag_filters(&["=value".to_string()]).unwrap_err(); + assert!(format!("{empty_key}").contains("--tag key must be non-empty")); + + let duplicate = parse_tag_filters(&[ + "persona=code-reviewer".to_string(), + "persona=builder".to_string(), + ]) + .unwrap_err(); + assert!(format!("{duplicate}").contains("duplicate --tag filter")); + } + + #[test] + fn grouped_json_includes_quality_when_report_has_it() { + let report = SummaryGroupedReport { + group_by: SummaryGroupBy::Model, + tag_key: None, + tag_values: Vec::new(), + turn_count: 0, + rows: Vec::new(), + total_cost: CostBreakdown { + model: String::new().into(), + total: 0.0, + input: 0.0, + output: 0.0, + reasoning: 0.0, + cache_read: 0.0, + cache_create: 0.0, + }, + fidelity: relayburn_sdk::summarize_fidelity(&[]), + per_cell_fidelity: json!({"groupBy": "model"}), + replacement_savings: relayburn_sdk::ReplacementSavingsSummary::default(), + stop_reasons: relayburn_sdk::StopReasonCounts::default(), + subagents: SubagentCounts::default(), + quality: Some(QualityResult::default()), + unpriced_turns: 0, + unpriced_models: Vec::new(), + }; + + let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty()); + + assert_eq!(value["quality"], json!({"outcomes": [], "oneShot": []})); + } + + #[test] + fn subagents_line_renders_only_when_counts_nonzero() { + // Empty bucket → skipped, line absent (so old summaries keep + // their byte-identical shape against the existing golden). + let empty = SubagentCounts::default(); + assert!(empty.is_empty()); + + // Non-zero bucket → human line includes both paired+orphan + // counts. Issue #435 explicitly wants both numbers even when + // one is zero, so a slash-command-only session showing only + // orphans is still scannable. + let counts = SubagentCounts { + paired: 2, + orphan: 1, + }; + assert_eq!( + format_subagents_line(&counts), + "subagents: 2 paired, 1 orphan" + ); + } + + #[test] + fn subagents_json_payload_includes_total_and_omits_when_empty() { + // Empty bucket → key absent in JSON so `summary.json | jq` for + // pre-#435 callers still passes without a `?.` guard. + let mut report = SummaryGroupedReport { + group_by: SummaryGroupBy::Model, + tag_key: None, + tag_values: Vec::new(), + turn_count: 0, + rows: Vec::new(), + total_cost: CostBreakdown { + model: String::new().into(), + total: 0.0, + input: 0.0, + output: 0.0, + reasoning: 0.0, + cache_read: 0.0, + cache_create: 0.0, + }, + fidelity: relayburn_sdk::summarize_fidelity(&[]), + per_cell_fidelity: json!({"groupBy": "model"}), + replacement_savings: relayburn_sdk::ReplacementSavingsSummary::default(), + stop_reasons: relayburn_sdk::StopReasonCounts::default(), + subagents: SubagentCounts::default(), + quality: None, + unpriced_turns: 0, + unpriced_models: Vec::new(), + }; + let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty()); + assert!( + value.get("subagents").is_none(), + "subagents key must be omitted when counts are zero; got {value}" + ); + + report.subagents = SubagentCounts { + paired: 2, + orphan: 1, + }; + let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty()); + assert_eq!( + value["subagents"], + json!({"paired": 2, "orphan": 1, "total": 3}), + "non-empty subagent counts must surface `paired`/`orphan`/`total`" + ); + } + + #[test] + fn partial_footer_names_gap_and_says_totals_include_all_turns() { + fn row(label: &str, coverage: relayburn_sdk::RowCoverage) -> UsageCostAggregateRow { + UsageCostAggregateRow { + label: label.to_string(), + turns: 0, + usage: relayburn_sdk::Usage::default(), + cost: CostBreakdown { + model: String::new().into(), + total: 0.0, + input: 0.0, + output: 0.0, + reasoning: 0.0, + cache_read: 0.0, + cache_create: 0.0, + }, + coverage, + } + } + + let mut claude_coverage = relayburn_sdk::RowCoverage::default(); + claude_coverage.input.known = 3; + claude_coverage.output.known = 3; + claude_coverage.reasoning.missing = 3; + + let mut codex_coverage = relayburn_sdk::RowCoverage::default(); + codex_coverage.input.known = 1; + codex_coverage.output.known = 1; + codex_coverage.reasoning.known = 1; + + assert_eq!( + format_partial_footer(&[ + row("claude-sonnet-4-6", claude_coverage), + row("gpt-5-codex", codex_coverage), + ]), + "* partial token coverage: largest gap is reasoning (missing on 3 of 4 turns); totals still include all turns", + ); + } + + #[test] + fn ingest_prelude_text_matches_human_banner() { + let report = relayburn_sdk::IngestReport { + ingested_sessions: 1, + appended_turns: 2_000, + ..relayburn_sdk::IngestReport::empty() + }; + + assert_eq!( + ingest_prelude_text(&report), + "\ningested 1 new session (+2,000 turns)\n", + ); + } +} diff --git a/crates/relayburn-sdk/src/analyze.rs b/crates/relayburn-sdk/src/analyze.rs index e4dce047..d1917272 100644 --- a/crates/relayburn-sdk/src/analyze.rs +++ b/crates/relayburn-sdk/src/analyze.rs @@ -38,67 +38,71 @@ pub mod tool_call_patterns; pub mod tool_output_bloat; mod util; -pub use claude_md::{ - build_trim_recommendations, render_unified_diff_for_recommendation, MarkdownSection, - SessionClaudeMdCost, TrimRecommendation, -}; -pub use compare::{ - build_compare_table, compare_from_archive, CompareCategory, CompareCell, - CompareFromArchiveResult, CompareOptions, CompareTable, CompareTotals, DEFAULT_MIN_SAMPLE, -}; +// Items below are split into `pub use` (mirrored at the SDK lib root) and +// `pub(crate) use` (internal-only; the verb layer reaches them via +// `crate::analyze::*`, but they are deliberately kept off the published +// surface). +pub(crate) use claude_md::{build_trim_recommendations, render_unified_diff_for_recommendation}; +pub use claude_md::{MarkdownSection, SessionClaudeMdCost}; +// The low-level compare building blocks are internal: the public compare +// surface is the `LedgerHandle::compare` / `compare_timeseries` verbs in +// `query_verbs::compare`, which wrap these. Only `DEFAULT_MIN_SAMPLE` stays +// public (the CLI uses it as the default `--min-sample`). +pub use compare::DEFAULT_MIN_SAMPLE; +pub(crate) use compare::{build_compare_table, CompareOptions, CompareTable}; +pub(crate) use context_delta::deltas_for_session; pub use context_delta::{ - deltas_for_session, ContextDelta, ContextDeltaOpts, InterveningStep, OwnerFilter, OwnerRail, - ReminderSource, + ContextDelta, ContextDeltaOpts, InterveningStep, OwnerFilter, OwnerRail, ReminderSource, }; -pub use cost::{cost_for_turn, cost_for_usage, sum_costs, tally_unpriced, CostBreakdown}; -pub use fidelity::{ - has_minimum_fidelity, summarize_fidelity, summarize_fidelity_from_iter, FidelitySummary, -}; -pub use findings::{findings_from_patterns, sort_findings, WasteFinding, WasteSeverity}; +pub(crate) use cost::sum_costs; +pub use cost::{cost_for_turn, tally_unpriced, CostBreakdown}; +pub(crate) use fidelity::has_minimum_fidelity; +pub use fidelity::{summarize_fidelity, summarize_fidelity_from_iter, FidelitySummary}; +pub(crate) use findings::findings_from_patterns; +pub use findings::{sort_findings, WasteFinding, WasteSeverity}; pub use flow_graph::{ flow_graph_from_trees, FlowEdge, FlowEdgeKind, FlowGraph, FlowNode, FlowNodeKind, FlowOpts, - TurnTokens, DEFAULT_MAX_TURNS as FLOW_DEFAULT_MAX_TURNS, INTER_TURN_GAP, RAIL_GAP, + TurnTokens, INTER_TURN_GAP, RAIL_GAP, }; pub use ghost_surface::{ detect_ghost_surface, ghost_surface_to_finding, GhostSurfaceFindingOptions, }; pub use ghost_surface_inputs::build_ghost_surface_inputs; -pub use hotspots::{ +pub(crate) use hotspots::{ aggregate_by_bash, aggregate_by_bash_verb, aggregate_by_file, aggregate_by_mcp_server, - aggregate_by_subagent, attribute_hotspots, AttributionMethod, BashAggregation, - BashVerbAggregation, FileAggregation, HotspotsOptions, HotspotsResult, McpServerAggregation, - SessionTotals, SubagentAggregation, ToolAttribution, + aggregate_by_subagent, attribute_hotspots, }; -pub use overhead::{ - attribute_overhead, describe_applies_to, find_overhead_files, load_overhead_file, - AttributeOverheadInput, OverheadAttribution, OverheadFile, OverheadFileAttribution, - OverheadFileKind, ParsedOverheadFile, -}; -pub use patterns::{detect_patterns, DetectPatternsOptions}; -pub use pricing::{load_pricing, ModelCost, PricingTable, ReasoningMode}; -pub use provider::{ - aggregate_by_provider, filter_turns_by_provider, filter_turns_by_provider_with_rules, - provider_for, AggregateByProviderOptions, AsTurnLike, CoverageField, FieldCoverage, - ProviderAggregateRow, ProviderFilter, RowCoverage, TurnProvider, UsageCostAggregateRow, +pub use hotspots::{ + AttributionMethod, BashAggregation, BashVerbAggregation, FileAggregation, HotspotsOptions, + McpServerAggregation, SubagentAggregation, }; -pub use provider_reattribution::ProviderRule; -pub use quality::{ - compute_quality, ComputeQualityOptions, OneShotMetrics, OutcomeLabel, QualityResult, - SessionOutcome, +pub(crate) use overhead::{ + attribute_overhead, find_overhead_files, load_overhead_file, AttributeOverheadInput, + OverheadAttribution, OverheadFile, ParsedOverheadFile, }; -pub use replacement_savings::{ - summarize_replacement_savings, ReplacementSavingsSummary, ToolSavingsAggregate, +pub use overhead::{describe_applies_to, OverheadFileKind}; +pub(crate) use patterns::detect_patterns; +pub use patterns::DetectPatternsOptions; +pub(crate) use pricing::{load_pricing, PricingTable}; +pub use pricing::{ModelCost, ReasoningMode}; +pub(crate) use provider::{ + aggregate_by_provider, AggregateByProviderOptions, ProviderAggregateRow, }; +pub(crate) use provider::{provider_for, ProviderFilter}; +pub use provider::{CoverageField, FieldCoverage, RowCoverage, UsageCostAggregateRow}; +pub(crate) use quality::{compute_quality, ComputeQualityOptions}; +pub use quality::{OneShotMetrics, OutcomeLabel, QualityResult, SessionOutcome}; +pub(crate) use replacement_savings::summarize_replacement_savings; +pub use replacement_savings::{ReplacementSavingsSummary, ToolSavingsAggregate}; pub use span_tree::{AttrValue, SpanEvent, SpanKind, SpanNode, SpanStatus, TurnSpanTree}; -pub use subagent_tree::{ - aggregate_subagent_type_stats, build_subagent_tree, BuildSubagentTreeOptions, SubagentTreeNode, - SubagentTypeStats, -}; -pub use tool_call_patterns::{ - detect_tool_call_patterns, tool_call_pattern_to_finding, DetectToolCallPatternsOptions, +pub(crate) use subagent_tree::{ + aggregate_subagent_type_stats, build_subagent_tree, BuildSubagentTreeOptions, }; +pub use subagent_tree::{SubagentTreeNode, SubagentTypeStats}; +pub(crate) use tool_call_patterns::detect_tool_call_patterns; +pub use tool_call_patterns::{tool_call_pattern_to_finding, DetectToolCallPatternsOptions}; +pub(crate) use tool_output_bloat::detect_tool_output_bloat; pub use tool_output_bloat::{ - detect_tool_output_bloat, load_claude_settings, project_claude_settings_path, - tool_output_bloat_to_finding, user_claude_settings_path, DetectToolOutputBloatOptions, - LoadedClaudeSettings, + load_claude_settings, project_claude_settings_path, tool_output_bloat_to_finding, + user_claude_settings_path, DetectToolOutputBloatOptions, LoadedClaudeSettings, }; diff --git a/crates/relayburn-sdk/src/analyze/claude_md.rs b/crates/relayburn-sdk/src/analyze/claude_md.rs index d782b2cf..5ac1d0a8 100644 --- a/crates/relayburn-sdk/src/analyze/claude_md.rs +++ b/crates/relayburn-sdk/src/analyze/claude_md.rs @@ -417,7 +417,7 @@ fn pick_dominant_model(counts: &IndexMap) -> String { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct TrimRecommendation { +pub(crate) struct TrimRecommendation { pub file_path: String, pub section: MarkdownSection, pub projected_savings_per_session: f64, @@ -425,7 +425,7 @@ pub struct TrimRecommendation { pub token_share: f64, } -pub fn build_trim_recommendations( +pub(crate) fn build_trim_recommendations( attribution: &ClaudeMdAttributionResult, top_n: usize, ) -> Vec { @@ -444,7 +444,7 @@ pub fn build_trim_recommendations( .collect() } -pub fn render_unified_diff_for_recommendation( +pub(crate) fn render_unified_diff_for_recommendation( file_path: &str, file_text: &str, rec: &TrimRecommendation, diff --git a/crates/relayburn-sdk/src/analyze/compare.rs b/crates/relayburn-sdk/src/analyze/compare.rs index 75b41d8a..b269f41c 100644 --- a/crates/relayburn-sdk/src/analyze/compare.rs +++ b/crates/relayburn-sdk/src/analyze/compare.rs @@ -11,18 +11,12 @@ use std::collections::BTreeMap; -use crate::ledger::{EnrichedTurn, Ledger, Query, Result as LedgerResult}; +use crate::ledger::EnrichedTurn; use crate::reader::ActivityCategory; use crate::analyze::cost::cost_for_turn; use crate::analyze::pricing::PricingTable; -/// Activity category label, or `"unclassified"` for turns the classifier -/// couldn't bucket. Mirrors the TS `ActivityCategory | "unclassified"` -/// union. Exposed as a [`String`] because callers consume it as a key into -/// [`CompareTable::cells`]. -pub type CompareCategory = String; - /// Default minimum sample count below which a non-empty cell is flagged as /// `insufficient_sample`. Matches the TS `DEFAULT_MIN_SAMPLE`. pub const DEFAULT_MIN_SAMPLE: u64 = 5; @@ -82,6 +76,7 @@ pub struct CompareOptions<'a> { } impl<'a> CompareOptions<'a> { + #[cfg(test)] pub fn new(pricing: &'a PricingTable) -> Self { Self { pricing, @@ -240,39 +235,6 @@ pub fn build_compare_table(turns: &[EnrichedTurn], opts: &CompareOptions<'_>) -> } } -/// Result of a [`compare_from_archive`] run. `analyzed_turns` is the -/// pre-`models`-filter turn count — matches the TS path's `analyzedTurns`, -/// which is sourced from `queryAll(q).length` rather than the post-filter -/// table. -#[derive(Debug, Clone, PartialEq)] -pub struct CompareFromArchiveResult { - pub table: CompareTable, - pub analyzed_turns: usize, -} - -/// Build a [`CompareTable`] sourced from the SQLite ledger. Thin shell over -/// [`Ledger::query_turns`] + [`build_compare_table`]: filters -/// (since / until / project / session_id / source) are applied inside -/// [`Ledger::query_turns`]; the model allow-list lives in `opts` and is -/// honored by [`build_compare_table`]. -/// -/// Per #259 the Rust ledger is SQLite-only, so the TS path's bespoke -/// per-column SQL aggregation collapses to this delegation. (Issue #347 -/// folded the previous `compare_archive` module here.) -pub fn compare_from_archive( - ledger: &Ledger, - q: &Query, - opts: &CompareOptions<'_>, -) -> LedgerResult { - let turns = ledger.query_turns(q)?; - let analyzed_turns = turns.len(); - let table = build_compare_table(&turns, opts); - Ok(CompareFromArchiveResult { - table, - analyzed_turns, - }) -} - fn to_cell(acc: Option<&Accum>, min_sample: u64) -> CompareCell { let Some(acc) = acc else { return empty_cell(); @@ -739,53 +701,6 @@ mod tests { assert!((sum - t.totals["claude-sonnet-4-6"].total_cost).abs() < 1e-9); } - #[test] - fn compare_from_archive_round_trips_through_ledger() { - use crate::ledger::{Ledger, LedgerLayout, Query}; - use tempfile::TempDir; - - let tmp = TempDir::new().unwrap(); - let layout = LedgerLayout::under(tmp.path()); - let mut ledger = Ledger::open(&layout.burn, &layout.content).unwrap(); - let pricing = load_builtin_pricing(); - - let mut records: Vec = Vec::new(); - for i in 0..3u64 { - records.push(TurnRecord { - v: 1, - source: SourceKind::ClaudeCode, - session_id: "s-1".into(), - session_path: None, - message_id: format!("m-{i}"), - turn_index: 0, - ts: format!("2026-04-20T00:00:0{i}.000Z"), - model: "claude-sonnet-4-6".into(), - project: None, - project_key: None, - usage: default_usage(), - tool_calls: Vec::::new(), - files_touched: None, - subagent: None, - stop_reason: None, - activity: Some(ActivityCategory::Coding), - retries: Some(0), - has_edits: Some(true), - fidelity: None, - }); - } - ledger.append_turns(&records).unwrap(); - - let opts = CompareOptions::new(&pricing); - let result = compare_from_archive(&ledger, &Query::default(), &opts).unwrap(); - - assert_eq!(result.analyzed_turns, 3); - assert_eq!(result.table.models, vec!["claude-sonnet-4-6"]); - assert!(result.table.categories.iter().any(|c| c == "coding")); - let cell = &result.table.cells["claude-sonnet-4-6"]["coding"]; - assert_eq!(cell.turns, 3); - assert_eq!(cell.edit_turns, 3); - } - #[test] fn computes_cache_hit_rate_across_input_and_cache_buckets() { let pricing = load_builtin_pricing(); diff --git a/crates/relayburn-sdk/src/analyze/context_delta.rs b/crates/relayburn-sdk/src/analyze/context_delta.rs index ce9a9baf..4a691131 100644 --- a/crates/relayburn-sdk/src/analyze/context_delta.rs +++ b/crates/relayburn-sdk/src/analyze/context_delta.rs @@ -267,7 +267,7 @@ impl ContextDeltaOpts { /// `curr` inference's model. Models the pricing table doesn't recognize /// charge `0.0` (matching the rest of the analyze surface, which never /// surfaces costs it can't price). -pub fn deltas_for_session( +pub(crate) fn deltas_for_session( trees: &[TurnSpanTree], compactions: &[CompactionEvent], pricing: &PricingTable, diff --git a/crates/relayburn-sdk/src/analyze/cost.rs b/crates/relayburn-sdk/src/analyze/cost.rs index fff140be..f4194d3b 100644 --- a/crates/relayburn-sdk/src/analyze/cost.rs +++ b/crates/relayburn-sdk/src/analyze/cost.rs @@ -20,7 +20,7 @@ use crate::analyze::provider_reattribution::{resolve_provider, strip_provider_pr pub struct CostBreakdown { /// Model identifier this breakdown is attributed to. Uses /// `Cow<'static, str>` so common labels like `"aggregate"` (from - /// [`sum_costs`]) can be carried as a `'static` borrow with no allocation, + /// `sum_costs`) can be carried as a `'static` borrow with no allocation, /// while per-turn breakdowns can still own a `String`. pub model: Cow<'static, str>, pub total: f64, @@ -45,7 +45,7 @@ pub struct CostForUsageOptions { /// analyze module. Prices in the pricing table are quoted per million tokens. pub(crate) const PER_MILLION: f64 = 1_000_000.0; -pub fn cost_for_usage( +pub(crate) fn cost_for_usage( usage: &Usage, model: &str, pricing: &PricingTable, @@ -180,7 +180,7 @@ pub fn tally_unpriced(turns: &[TurnRecord], pricing: &PricingTable) -> (u64, Vec (count, models) } -pub fn sum_costs(costs: I) -> CostBreakdown +pub(crate) fn sum_costs(costs: I) -> CostBreakdown where I: IntoIterator, B: std::borrow::Borrow, diff --git a/crates/relayburn-sdk/src/analyze/fidelity.rs b/crates/relayburn-sdk/src/analyze/fidelity.rs index ce1344d1..11f509ca 100644 --- a/crates/relayburn-sdk/src/analyze/fidelity.rs +++ b/crates/relayburn-sdk/src/analyze/fidelity.rs @@ -4,7 +4,7 @@ //! Higher-level aggregators (compare, hotspots) use this to refuse stats on //! undersized fixtures, so it lands before them. -use std::collections::HashMap; +use std::collections::BTreeMap; use serde::Serialize; @@ -54,29 +54,29 @@ fn coverage_get(c: &Coverage, field: &str) -> bool { #[serde(rename_all = "camelCase")] pub struct FidelitySummary { pub total: u64, - pub by_class: HashMap, - pub by_granularity: HashMap, - pub missing_coverage: HashMap<&'static str, u64>, + pub by_class: BTreeMap, + pub by_granularity: BTreeMap, + pub missing_coverage: BTreeMap<&'static str, u64>, /// Records with no `fidelity` field at all — emitted by older ledger /// writers. Counted separately so we don't pretend they're "full". pub unknown: u64, } pub fn empty_fidelity_summary() -> FidelitySummary { - let mut by_class = HashMap::new(); + let mut by_class = BTreeMap::new(); by_class.insert(FidelityClass::Full, 0); by_class.insert(FidelityClass::UsageOnly, 0); by_class.insert(FidelityClass::AggregateOnly, 0); by_class.insert(FidelityClass::CostOnly, 0); by_class.insert(FidelityClass::Partial, 0); - let mut by_granularity = HashMap::new(); + let mut by_granularity = BTreeMap::new(); by_granularity.insert(UsageGranularity::PerTurn, 0); by_granularity.insert(UsageGranularity::PerMessage, 0); by_granularity.insert(UsageGranularity::PerSessionAggregate, 0); by_granularity.insert(UsageGranularity::CostOnly, 0); - let mut missing_coverage: HashMap<&'static str, u64> = HashMap::new(); + let mut missing_coverage: BTreeMap<&'static str, u64> = BTreeMap::new(); for field in COVERAGE_FIELDS { missing_coverage.insert(*field, 0); } diff --git a/crates/relayburn-sdk/src/analyze/findings.rs b/crates/relayburn-sdk/src/analyze/findings.rs index 4998b522..aa67e093 100644 --- a/crates/relayburn-sdk/src/analyze/findings.rs +++ b/crates/relayburn-sdk/src/analyze/findings.rs @@ -632,7 +632,7 @@ Total cost {cost}.", /// Roll the full PatternsResult into a single severity-ranked list. Within /// the same severity tier, sort by `usdPerSession` descending so the most /// expensive findings surface first. -pub fn findings_from_patterns(result: &PatternsResult) -> Vec { +pub(crate) fn findings_from_patterns(result: &PatternsResult) -> Vec { let mut findings: Vec = Vec::new(); for r in &result.retry_loops { findings.push(retry_loop_to_finding(r)); diff --git a/crates/relayburn-sdk/src/analyze/flow_graph.rs b/crates/relayburn-sdk/src/analyze/flow_graph.rs index ece0d19c..50fca12d 100644 --- a/crates/relayburn-sdk/src/analyze/flow_graph.rs +++ b/crates/relayburn-sdk/src/analyze/flow_graph.rs @@ -55,7 +55,7 @@ pub const RAIL_GAP: i32 = 32; /// Default `--max-turns` cap — see the CLI surface. Layouts wider than /// ~50 columns get unreadable in static SVG / Mermaid; embedders that /// want the full session can pass [`FlowOpts::max_turns`] explicitly. -pub const DEFAULT_MAX_TURNS: u32 = 50; +pub(crate) const DEFAULT_MAX_TURNS: u32 = 50; /// What kind of node a [`FlowNode`] represents in the flow DAG. Mirrors /// agent-profiler's node-kind registry so embedders that ship their own diff --git a/crates/relayburn-sdk/src/analyze/hotspots.rs b/crates/relayburn-sdk/src/analyze/hotspots.rs index 1cf6a95d..d56d5624 100644 --- a/crates/relayburn-sdk/src/analyze/hotspots.rs +++ b/crates/relayburn-sdk/src/analyze/hotspots.rs @@ -50,7 +50,7 @@ pub enum AttributionMethod { /// turns' `cacheRead`. `total_cost` is `initial_cost + persistence_cost`. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct ToolAttribution { +pub(crate) struct ToolAttribution { pub tool_use_id: String, pub tool_name: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -97,7 +97,7 @@ pub struct ToolAttribution { /// the 1e-9 USD precision contract. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct SessionTotals { +pub(crate) struct SessionTotals { pub session_id: String, pub grand_cost: f64, pub attributed_cost: f64, @@ -270,7 +270,10 @@ static FILE_TOOLS: phf::Set<&'static str> = phf_set! { /// `grand_total` and per-session `grand_cost` route through `cost_for_turn`, /// so anything outside the attributable surface (system prompts, reasoning /// charged via Codex `included_in_output`, etc.) lands in `unattributed_*`. -pub fn attribute_hotspots(turns: &[TurnRecord], opts: &HotspotsOptions<'_>) -> HotspotsResult { +pub(crate) fn attribute_hotspots( + turns: &[TurnRecord], + opts: &HotspotsOptions<'_>, +) -> HotspotsResult { // First-seen session ordering matches the TS `Map` iteration semantics. // Borrow turns rather than cloning — nothing below mutates them and the // input slice outlives every aggregation step. @@ -685,7 +688,7 @@ fn accumulate_output_bytes( /// Roll up file-touching tool attributions (`Read | Edit | Write | /// NotebookEdit`) by their target path. Rows missing or with an empty target /// are skipped. Output is sorted by `total_cost` descending. -pub fn aggregate_by_file(attributions: &[ToolAttribution]) -> Vec { +pub(crate) fn aggregate_by_file(attributions: &[ToolAttribution]) -> Vec { aggregate( attributions, |a| { @@ -735,7 +738,7 @@ pub fn aggregate_by_file(attributions: &[ToolAttribution]) -> Vec Vec { +pub(crate) fn aggregate_by_bash(attributions: &[ToolAttribution]) -> Vec { aggregate( attributions, |a| (a.tool_name == "Bash").then(|| a.args_hash.clone()), @@ -800,7 +803,7 @@ struct BashVerbExample { /// up to three highest-cost representative commands (cost desc, then command /// asc as tiebreaker). Output is sorted by `total_cost` desc, then `verb` /// asc. -pub fn aggregate_by_bash_verb( +pub(crate) fn aggregate_by_bash_verb( attributions: &[ToolAttribution], parse: F, ) -> Vec @@ -914,7 +917,7 @@ fn parse_mcp_tool_name(name: &str) -> Option<(&str, &str)> { /// Roll up `Agent` / `Task` spawn attributions by `subagent_type`. Spawns /// without a resolved type bucket under `"(unknown)"`. Output is sorted by /// `total_cost` descending. -pub fn aggregate_by_subagent(attributions: &[ToolAttribution]) -> Vec { +pub(crate) fn aggregate_by_subagent(attributions: &[ToolAttribution]) -> Vec { aggregate( attributions, |a| { @@ -971,7 +974,9 @@ struct McpServerAccumulator { /// tools (and malformed `mcp__…` names that fail to split into a /// non-empty server + tool) are skipped. Output is sorted by `total_cost` /// desc, then `server` asc as a stable tiebreaker. -pub fn aggregate_by_mcp_server(attributions: &[ToolAttribution]) -> Vec { +pub(crate) fn aggregate_by_mcp_server( + attributions: &[ToolAttribution], +) -> Vec { let mut by_server: IndexMap = IndexMap::new(); for a in attributions { let Some((server, tool)) = parse_mcp_tool_name(&a.tool_name) else { diff --git a/crates/relayburn-sdk/src/analyze/overhead.rs b/crates/relayburn-sdk/src/analyze/overhead.rs index cbcb0e17..b11986b2 100644 --- a/crates/relayburn-sdk/src/analyze/overhead.rs +++ b/crates/relayburn-sdk/src/analyze/overhead.rs @@ -27,7 +27,7 @@ pub enum OverheadFileKind { } #[derive(Debug, Clone, PartialEq)] -pub struct OverheadFile { +pub(crate) struct OverheadFile { pub kind: OverheadFileKind, pub path: String, /// Which agent sources read this file into their cached context. A turn's @@ -36,20 +36,20 @@ pub struct OverheadFile { } #[derive(Debug, Clone, PartialEq)] -pub struct ParsedOverheadFile { +pub(crate) struct ParsedOverheadFile { pub file: OverheadFile, pub parsed: ParsedClaudeMd, } #[derive(Debug, Clone, PartialEq)] -pub struct OverheadFileAttribution { +pub(crate) struct OverheadFileAttribution { pub file: OverheadFile, pub parsed: ParsedClaudeMd, pub attribution: ClaudeMdAttributionResult, } #[derive(Debug, Clone, PartialEq)] -pub struct OverheadAttribution { +pub(crate) struct OverheadAttribution { pub per_file: Vec, pub grand_total: f64, /// Count of distinct turns that contributed to at least one file's cost. @@ -59,7 +59,7 @@ pub struct OverheadAttribution { pub total_riding_turns: u64, } -pub struct AttributeOverheadInput<'a> { +pub(crate) struct AttributeOverheadInput<'a> { pub files: &'a [ParsedOverheadFile], pub turns: &'a [TurnRecord], pub pricing: &'a PricingTable, @@ -89,7 +89,7 @@ const CANDIDATES: &[Candidate] = &[ }, ]; -pub fn find_overhead_files(project_path: &Path) -> Vec { +pub(crate) fn find_overhead_files(project_path: &Path) -> Vec { let mut out = Vec::new(); for c in CANDIDATES { let mut abs = project_path.to_path_buf(); @@ -110,12 +110,12 @@ pub fn find_overhead_files(project_path: &Path) -> Vec { out } -pub fn load_overhead_file(file: OverheadFile) -> std::io::Result { +pub(crate) fn load_overhead_file(file: OverheadFile) -> std::io::Result { let parsed = load_claude_md_file(Path::new(&file.path))?; Ok(ParsedOverheadFile { file, parsed }) } -pub fn attribute_overhead(input: AttributeOverheadInput<'_>) -> OverheadAttribution { +pub(crate) fn attribute_overhead(input: AttributeOverheadInput<'_>) -> OverheadAttribution { let mut per_file: Vec = Vec::new(); // Per-session max riding-turns across every file. The eviction check is // `cache_read >= file_tokens`, so a smaller file's rides are a strict diff --git a/crates/relayburn-sdk/src/analyze/patterns.rs b/crates/relayburn-sdk/src/analyze/patterns.rs index 296e3ee6..2eeafe80 100644 --- a/crates/relayburn-sdk/src/analyze/patterns.rs +++ b/crates/relayburn-sdk/src/analyze/patterns.rs @@ -125,23 +125,12 @@ pub struct DetectPatternsOptions<'a> { pub tool_result_events: Option<&'a [ToolResultEventRecord]>, } -impl<'a> DetectPatternsOptions<'a> { - /// Convenience constructor used by tests and embedders that only need - /// to supply pricing. - pub fn with_pricing(pricing: &'a PricingTable) -> Self { - Self { - pricing, - compactions: None, - user_turns_by_session: None, - content_by_session: None, - tool_result_events: None, - } - } -} - /// Run every detector across the supplied turn stream. Mirrors the TS /// `detectPatterns` orchestrator (patterns.ts:273-345). -pub fn detect_patterns(turns: &[TurnRecord], opts: &DetectPatternsOptions<'_>) -> PatternsResult { +pub(crate) fn detect_patterns( + turns: &[TurnRecord], + opts: &DetectPatternsOptions<'_>, +) -> PatternsResult { let by_session = group_turns_by_session_sorted(turns); let events_by_session = group_tool_result_events_by_session(opts.tool_result_events); diff --git a/crates/relayburn-sdk/src/analyze/pricing.rs b/crates/relayburn-sdk/src/analyze/pricing.rs index b6e4d1d3..39b2d7e8 100644 --- a/crates/relayburn-sdk/src/analyze/pricing.rs +++ b/crates/relayburn-sdk/src/analyze/pricing.rs @@ -46,7 +46,7 @@ pub struct ModelCost { pub reasoning_mode: ReasoningMode, } -pub type PricingTable = HashMap; +pub(crate) type PricingTable = HashMap; #[derive(Debug, Default, Deserialize)] struct ModelsDevModel { diff --git a/crates/relayburn-sdk/src/analyze/provider.rs b/crates/relayburn-sdk/src/analyze/provider.rs index ae22f498..c2117370 100644 --- a/crates/relayburn-sdk/src/analyze/provider.rs +++ b/crates/relayburn-sdk/src/analyze/provider.rs @@ -24,7 +24,7 @@ use crate::analyze::provider_reattribution::{ }; #[derive(Debug, Clone, PartialEq, Eq)] -pub struct TurnProvider { +pub(crate) struct TurnProvider { pub provider: String, pub raw_model: String, pub normalized_model: String, @@ -105,7 +105,7 @@ pub struct UsageCostAggregateRow { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct ProviderAggregateRow { +pub(crate) struct ProviderAggregateRow { pub provider: String, pub label: String, pub turns: u64, @@ -115,7 +115,7 @@ pub struct ProviderAggregateRow { } #[derive(Debug, Clone, Copy)] -pub struct AggregateByProviderOptions<'a> { +pub(crate) struct AggregateByProviderOptions<'a> { pub pricing: &'a PricingTable, /// `None` defers to [`default_rules`]. pub rules: Option<&'a [ProviderRule]>, @@ -180,7 +180,8 @@ pub fn provider_for_model_with_rules( /// Filter turns to those whose effective provider (lower-cased) is in /// `filter`. Returns the input slice unchanged when `filter` is `None`. -pub fn filter_turns_by_provider<'a, T>( +#[cfg(test)] +pub(crate) fn filter_turns_by_provider<'a, T>( turns: &'a [T], filter: Option<&ProviderFilter>, ) -> Vec<&'a T> @@ -190,7 +191,8 @@ where filter_turns_by_provider_with_rules(turns, filter, default_rules()) } -pub fn filter_turns_by_provider_with_rules<'a, T>( +#[cfg(test)] +pub(crate) fn filter_turns_by_provider_with_rules<'a, T>( turns: &'a [T], filter: Option<&ProviderFilter>, rules: &[ProviderRule], @@ -215,11 +217,13 @@ where /// 'source'>`. Implemented for [`TurnRecord`] out of the box; downstream /// callers can implement it for their own row types if they want to share /// [`filter_turns_by_provider`]. -pub trait AsTurnLike { +#[cfg(test)] +pub(crate) trait AsTurnLike { fn model_str(&self) -> &str; fn source_kind(&self) -> SourceKind; } +#[cfg(test)] impl AsTurnLike for TurnRecord { fn model_str(&self) -> &str { &self.model @@ -229,7 +233,7 @@ impl AsTurnLike for TurnRecord { } } -pub fn aggregate_by_provider( +pub(crate) fn aggregate_by_provider( turns: &[TurnRecord], opts: AggregateByProviderOptions<'_>, ) -> Vec { diff --git a/crates/relayburn-sdk/src/analyze/provider_reattribution.rs b/crates/relayburn-sdk/src/analyze/provider_reattribution.rs index 554bfb9a..29317e31 100644 --- a/crates/relayburn-sdk/src/analyze/provider_reattribution.rs +++ b/crates/relayburn-sdk/src/analyze/provider_reattribution.rs @@ -38,12 +38,15 @@ pub struct ProviderResolution { /// capturing group whose first capture is the normalized model id. #[derive(Debug, Clone)] pub enum ProviderPattern { + // Constructed only via the test-only `ProviderRule::prefix` helper, but + // matched by the always-compiled `apply_rule` resolver below. + #[cfg_attr(not(test), allow(dead_code))] Prefix(String), Regex(Regex), } #[derive(Debug, Clone)] -pub struct ProviderRule { +pub(crate) struct ProviderRule { /// Stable identifier; appears in [`ProviderResolution::matched_rule`] and /// is used to dedupe / replace rules when callers extend /// [`default_rules`]. @@ -59,7 +62,8 @@ pub struct ProviderRule { impl ProviderRule { /// Build a literal-prefix rule. The rule fires when the model string /// starts with `prefix`; the normalized model is the residual after the - /// prefix. + /// prefix. Test-only since the published custom-rule surface was retired. + #[cfg(test)] pub fn prefix( name: impl Into, provider: impl Into, @@ -71,20 +75,6 @@ impl ProviderRule { pattern: ProviderPattern::Prefix(prefix.into()), } } - - /// Build a regex-pattern rule. Returns an error if `pattern` doesn't - /// compile. - pub fn regex( - name: impl Into, - provider: impl Into, - pattern: &str, - ) -> Result { - Ok(Self { - name: name.into(), - provider: provider.into(), - pattern: ProviderPattern::Regex(Regex::new(pattern)?), - }) - } } // Rule order matters: the first match wins. More specific prefixes diff --git a/crates/relayburn-sdk/src/analyze/quality.rs b/crates/relayburn-sdk/src/analyze/quality.rs index 80e5a0e8..4feacb1f 100644 --- a/crates/relayburn-sdk/src/analyze/quality.rs +++ b/crates/relayburn-sdk/src/analyze/quality.rs @@ -84,7 +84,7 @@ pub struct QualityResult { } #[derive(Debug, Clone, Default)] -pub struct ComputeQualityOptions<'a> { +pub(crate) struct ComputeQualityOptions<'a> { /// Optional content sidecar records. When provided, give-up phrase /// matching on the last assistant text downgrades assistant-ended /// sessions from `completed/medium` to `completed/low`. Without content, @@ -111,7 +111,7 @@ const SHORT_CONVERSATION_THRESHOLD: usize = 3; const LONG_CONVERSATION_THRESHOLD: usize = 10; const FAILURE_STREAK_THRESHOLD: u64 = 3; -pub fn compute_quality(turns: &[TurnRecord], opts: &ComputeQualityOptions) -> QualityResult { +pub(crate) fn compute_quality(turns: &[TurnRecord], opts: &ComputeQualityOptions) -> QualityResult { let by_session = group_turns_by_session_sorted(turns); let now = opts.now_ms.unwrap_or_else(now_ms_system); diff --git a/crates/relayburn-sdk/src/analyze/replacement_savings.rs b/crates/relayburn-sdk/src/analyze/replacement_savings.rs index 00ebb4c6..5173d6dd 100644 --- a/crates/relayburn-sdk/src/analyze/replacement_savings.rs +++ b/crates/relayburn-sdk/src/analyze/replacement_savings.rs @@ -141,7 +141,7 @@ pub fn estimate_savings_for_tool_call( }) } -pub fn summarize_replacement_savings( +pub(crate) fn summarize_replacement_savings( turns: &[TurnRecord], options: Option<&ReplacementSavingsOptions>, ) -> ReplacementSavingsSummary { diff --git a/crates/relayburn-sdk/src/analyze/subagent_tree.rs b/crates/relayburn-sdk/src/analyze/subagent_tree.rs index 85fcad48..9bf39dab 100644 --- a/crates/relayburn-sdk/src/analyze/subagent_tree.rs +++ b/crates/relayburn-sdk/src/analyze/subagent_tree.rs @@ -36,7 +36,7 @@ pub struct SubagentTreeNode { } #[derive(Debug, Clone)] -pub struct BuildSubagentTreeOptions<'a> { +pub(crate) struct BuildSubagentTreeOptions<'a> { pub pricing: &'a PricingTable, pub relationships: Option<&'a [SessionRelationshipRecord]>, } @@ -60,7 +60,7 @@ impl<'a> BuildSubagentTreeOptions<'a> { /// `subagent.agentId`), nested by `parentAgentId`. When relationship rows are /// supplied, they are the primary substrate; per-turn `subagent` fields /// attach turn cost and fill legacy gaps. -pub fn build_subagent_tree( +pub(crate) fn build_subagent_tree( turns: &[TurnRecord], opts: &BuildSubagentTreeOptions<'_>, ) -> IndexMap { @@ -553,7 +553,7 @@ pub struct SubagentTypeStats { /// Aggregate subagent invocations across sessions by `subagentType`. An /// invocation is the unique `(sessionId, agentId)` pair so the same agent id /// re-used across sessions doesn't collide. -pub fn aggregate_subagent_type_stats( +pub(crate) fn aggregate_subagent_type_stats( turns: &[TurnRecord], opts: &BuildSubagentTreeOptions<'_>, ) -> Vec { diff --git a/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs b/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs index a2815c8f..e7b8f2ff 100644 --- a/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs +++ b/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs @@ -68,7 +68,7 @@ static BASH_RAW_NAMES: phf::Set<&'static str> = phf_set! { "Bash", "bash", "exec_command", "shell" }; -pub fn detect_tool_call_patterns( +pub(crate) fn detect_tool_call_patterns( turns: &[TurnRecord], opts: &DetectToolCallPatternsOptions<'_>, ) -> Vec { diff --git a/crates/relayburn-sdk/src/analyze/tool_output_bloat.rs b/crates/relayburn-sdk/src/analyze/tool_output_bloat.rs index f80c52ed..876414d5 100644 --- a/crates/relayburn-sdk/src/analyze/tool_output_bloat.rs +++ b/crates/relayburn-sdk/src/analyze/tool_output_bloat.rs @@ -443,21 +443,9 @@ pub struct DetectToolOutputBloatOptions<'a> { pub min_occurrences: Option, } -impl<'a> DetectToolOutputBloatOptions<'a> { - pub fn new(pricing: &'a PricingTable) -> Self { - Self { - settings: &[], - tool_result_events: &[], - user_turns: &[], - turns: &[], - pricing, - threshold: None, - min_occurrences: None, - } - } -} - -pub fn detect_tool_output_bloat(opts: &DetectToolOutputBloatOptions<'_>) -> Vec { +pub(crate) fn detect_tool_output_bloat( + opts: &DetectToolOutputBloatOptions<'_>, +) -> Vec { let mut out: Vec = Vec::new(); if !opts.settings.is_empty() { out.extend(detect_static_config_bloat( diff --git a/crates/relayburn-sdk/src/lib.rs b/crates/relayburn-sdk/src/lib.rs index 59fa6ba9..0455ad1e 100644 --- a/crates/relayburn-sdk/src/lib.rs +++ b/crates/relayburn-sdk/src/lib.rs @@ -88,32 +88,24 @@ pub use crate::ledger::{ }; pub use crate::analyze::{ - deltas_for_session, ContextDelta, ContextDeltaOpts, InterveningStep, - OwnerFilter as ContextDeltaOwnerFilter, OwnerRail as ContextDeltaOwnerRail, ReminderSource, + ContextDelta, ContextDeltaOpts, InterveningStep, OwnerFilter as ContextDeltaOwnerFilter, + OwnerRail as ContextDeltaOwnerRail, ReminderSource, }; +// NOTE: the low-level `compare` building blocks (`build_compare_table`, +// `CompareTable` / `CompareCell` / `CompareTotals`, and the +// `CompareOptions` aliased internally as `AnalyzeCompareOptions`) and the +// helpers `load_pricing` / `provider_for` / `has_minimum_fidelity` / +// `ProviderFilter` are deliberately NOT re-exported — they are `pub(crate)` +// internals of the compare verb. The public compare surface is the +// `LedgerHandle::compare` / `compare_timeseries` verbs (see `query_verbs`). pub use crate::analyze::{ - aggregate_by_bash, aggregate_by_bash_verb, aggregate_by_file, aggregate_by_mcp_server, - aggregate_by_provider, aggregate_by_subagent, aggregate_subagent_type_stats, - attribute_hotspots, attribute_overhead, build_compare_table, build_subagent_tree, - build_trim_recommendations, compare_from_archive, compute_quality, cost_for_turn, - cost_for_usage, describe_applies_to, detect_patterns, detect_tool_call_patterns, - detect_tool_output_bloat, filter_turns_by_provider, filter_turns_by_provider_with_rules, - find_overhead_files, findings_from_patterns, has_minimum_fidelity, load_overhead_file, - load_pricing, provider_for, render_unified_diff_for_recommendation, sum_costs, - summarize_fidelity, summarize_replacement_savings, AggregateByProviderOptions, AsTurnLike, - AttributeOverheadInput, AttributionMethod, BashAggregation, BashVerbAggregation, - BuildSubagentTreeOptions, CompareCategory, CompareCell, CompareFromArchiveResult, - CompareOptions as AnalyzeCompareOptions, CompareTable, CompareTotals, ComputeQualityOptions, - CostBreakdown, CoverageField, FidelitySummary, FieldCoverage, FileAggregation, - HotspotsOptions as AnalyzeHotspotsOptions, HotspotsResult as AnalyzeHotspotsResult, - MarkdownSection, McpServerAggregation, ModelCost, OneShotMetrics, OutcomeLabel, - OverheadAttribution, OverheadFile, OverheadFileAttribution, OverheadFileKind, - ParsedOverheadFile, PricingTable, ProviderAggregateRow, ProviderFilter, ProviderRule, - QualityResult, ReasoningMode, ReplacementSavingsSummary, RowCoverage, SessionClaudeMdCost, - SessionOutcome, SessionTotals, SubagentAggregation, SubagentTreeNode, SubagentTypeStats, - ToolAttribution, TrimRecommendation, TurnProvider, UsageCostAggregateRow, WasteFinding, - WasteSeverity, DEFAULT_MIN_SAMPLE, + cost_for_turn, describe_applies_to, summarize_fidelity, AttributionMethod, BashAggregation, + BashVerbAggregation, CostBreakdown, CoverageField, FidelitySummary, FieldCoverage, + FileAggregation, MarkdownSection, McpServerAggregation, ModelCost, OneShotMetrics, + OutcomeLabel, OverheadFileKind, QualityResult, ReasoningMode, ReplacementSavingsSummary, + RowCoverage, SessionClaudeMdCost, SessionOutcome, SubagentAggregation, SubagentTreeNode, + SubagentTypeStats, UsageCostAggregateRow, WasteFinding, WasteSeverity, DEFAULT_MIN_SAMPLE, }; // Span tree primitives (issue #430). Re-exported at the SDK root so @@ -128,7 +120,7 @@ pub use crate::analyze::{ // span tree. pub use crate::analyze::{ flow_graph_from_trees, FlowEdge, FlowEdgeKind, FlowGraph, FlowNode, FlowNodeKind, FlowOpts, - TurnTokens as FlowTurnTokens, FLOW_DEFAULT_MAX_TURNS, INTER_TURN_GAP, RAIL_GAP, + TurnTokens as FlowTurnTokens, INTER_TURN_GAP, RAIL_GAP, }; pub use crate::ingest::{ diff --git a/crates/relayburn-sdk/src/query_verbs/compare.rs b/crates/relayburn-sdk/src/query_verbs/compare.rs index 52ad3c5b..7b689151 100644 --- a/crates/relayburn-sdk/src/query_verbs/compare.rs +++ b/crates/relayburn-sdk/src/query_verbs/compare.rs @@ -371,7 +371,13 @@ fn fidelity_rank(class: FidelityClass) -> u8 { } } +/// Round to `digits` decimal places matching JS `Number(n.toFixed(digits))` / +/// Rust `format!("{n:.digits$}")` semantics (round half-to-even on the decimal +/// string), rather than `f64::round`'s half-away-from-zero. The presenter +/// layer re-formats these values with `format!("{:.N}")`, so rounding the same +/// way here keeps that second pass idempotent — at exact ties the two +/// rounding modes otherwise disagree in the last digit. fn round_digits(n: f64, digits: i32) -> f64 { - let scale = 10_f64.powi(digits); - (n * scale).round() / scale + let s = format!("{n:.*}", digits.max(0) as usize); + s.parse().unwrap_or(n) } diff --git a/crates/relayburn-sdk/src/query_verbs/mod.rs b/crates/relayburn-sdk/src/query_verbs/mod.rs index 0fb75abd..cf58e658 100644 --- a/crates/relayburn-sdk/src/query_verbs/mod.rs +++ b/crates/relayburn-sdk/src/query_verbs/mod.rs @@ -9,44 +9,46 @@ //! Option` so callers don't have to mutate process env to point //! at a non-default ledger. -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; use anyhow::Result; -use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use crate::analyze::{ aggregate_by_bash, aggregate_by_bash_verb, aggregate_by_file, aggregate_by_mcp_server, aggregate_by_provider, aggregate_by_subagent, aggregate_subagent_type_stats, attribute_hotspots, attribute_overhead, build_compare_table, build_ghost_surface_inputs, - build_subagent_tree, build_trim_recommendations, compute_quality, cost_for_turn, - deltas_for_session, detect_ghost_surface, detect_patterns, detect_tool_call_patterns, - detect_tool_output_bloat, find_overhead_files, findings_from_patterns, - ghost_surface_to_finding, has_minimum_fidelity, load_claude_settings, load_overhead_file, - load_pricing, project_claude_settings_path, provider_for, + build_subagent_tree, build_trim_recommendations, cost_for_turn, deltas_for_session, + detect_ghost_surface, detect_patterns, detect_tool_call_patterns, detect_tool_output_bloat, + find_overhead_files, findings_from_patterns, ghost_surface_to_finding, has_minimum_fidelity, + load_claude_settings, load_overhead_file, load_pricing, project_claude_settings_path, render_unified_diff_for_recommendation, sort_findings, sum_costs, summarize_fidelity, summarize_fidelity_from_iter, summarize_replacement_savings, tally_unpriced, tool_call_pattern_to_finding, tool_output_bloat_to_finding, user_claude_settings_path, AggregateByProviderOptions, AttributeOverheadInput, AttributionMethod, BashAggregation, BashVerbAggregation, BuildSubagentTreeOptions, CompareOptions as AnalyzeCompareOptions, - CompareTable, ComputeQualityOptions, ContextDelta, ContextDeltaOpts, CostBreakdown, - CoverageField, DetectPatternsOptions, DetectToolCallPatternsOptions, - DetectToolOutputBloatOptions, FidelitySummary, FieldCoverage, FileAggregation, + CompareTable, ContextDelta, ContextDeltaOpts, CostBreakdown, DetectPatternsOptions, + DetectToolCallPatternsOptions, DetectToolOutputBloatOptions, FidelitySummary, FileAggregation, GhostSurfaceFindingOptions, HotspotsOptions as AnalyzeHotspotsOptions, LoadedClaudeSettings, MarkdownSection, McpServerAggregation, OverheadFile, OverheadFileKind, OwnerRail, - ParsedOverheadFile, PricingTable, ProviderAggregateRow, ProviderFilter, QualityResult, - ReplacementSavingsSummary, RowCoverage, SessionClaudeMdCost, SubagentAggregation, - SubagentTreeNode, SubagentTypeStats, ToolSavingsAggregate, TurnSpanTree, UsageCostAggregateRow, - WasteFinding, + ParsedOverheadFile, PricingTable, ProviderFilter, QualityResult, ReplacementSavingsSummary, + SessionClaudeMdCost, SubagentAggregation, SubagentTreeNode, SubagentTypeStats, + ToolSavingsAggregate, TurnSpanTree, UsageCostAggregateRow, WasteFinding, }; use crate::ledger::{EnrichedTurn, Enrichment, Query}; use crate::reader::{ - parse_bash_command, resolve_project, BashParse, ContentRecord, Coverage, FidelityClass, - RelationshipType, SessionRelationshipRecord, SourceKind, StopReason, TurnRecord, Usage, - UsageGranularity, UserTurnBlockKind, UserTurnRecord, + parse_bash_command, resolve_project, BashParse, FidelityClass, RelationshipType, SourceKind, + StopReason, TurnRecord, UsageGranularity, UserTurnRecord, }; +// Re-exported only for the `tests` submodule, which reaches these names +// through `use super::*`. The non-test query-verb code no longer references +// them directly since the summary compute engine moved into `summary/compute`. +#[cfg(test)] +use crate::reader::SessionRelationshipRecord; +#[cfg(test)] +use indexmap::IndexMap; use crate::{Ledger, LedgerHandle, LedgerOpenOptions}; diff --git a/crates/relayburn-sdk/src/query_verbs/summary.rs b/crates/relayburn-sdk/src/query_verbs/summary.rs deleted file mode 100644 index 5ac6c712..00000000 --- a/crates/relayburn-sdk/src/query_verbs/summary.rs +++ /dev/null @@ -1,2056 +0,0 @@ -use super::*; - -// --------------------------------------------------------------------------- -// summary -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryOptions { - pub session: Option, - pub project: Option, - pub since: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tags: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub group_by_tag: Option, - pub ledger_home: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryToolRow { - pub tool: String, - pub tokens: u64, - pub cost: f64, - pub count: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryModelRow { - pub model: String, - pub tokens: u64, - pub cost: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryTagRow { - pub tag: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub value: Option, - pub tokens: u64, - pub cost: f64, - pub turn_count: u64, -} - -/// Per-outcome turn counts, surfaced by `burn summary` for the one-line -/// outcome breakdown (`142 end_turn, 3 max_tokens, 1 refusal, 0 pause`). -/// -/// Counts mirror the [`StopReason`] enum variants plus a `none` slot for -/// turns whose row carried no `stop_reason` field at all — that's Codex -/// today (no field in the rollout schema) and any pre-3.0 ledger row that -/// was ingested before the reader started populating the enum. -/// -/// `Silent` is reserved for "row exists, carries a stop_reason that we -/// don't recognize" — distinct from `none` so we can spot a future harness -/// regression rather than silently lumping it with Codex. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct StopReasonCounts { - pub end_turn: u64, - pub max_tokens: u64, - pub pause_turn: u64, - pub stop_sequence: u64, - pub tool_use: u64, - pub refusal: u64, - pub silent: u64, - /// Turns whose record carried no `stop_reason` field — e.g. Codex - /// rollouts (the harness doesn't report one) or pre-3.0 ledger rows - /// from before the reader started parsing the field. - pub none: u64, -} - -impl StopReasonCounts { - /// Accumulate one turn's outcome into the bucket counts. `None` lands - /// in [`Self::none`]; unrecognized variants would already be normalized - /// to [`StopReason::Silent`] upstream by the lenient deserializer. - pub fn bump(&mut self, reason: Option) { - match reason { - None => self.none += 1, - Some(StopReason::EndTurn) => self.end_turn += 1, - Some(StopReason::MaxTokens) => self.max_tokens += 1, - Some(StopReason::PauseTurn) => self.pause_turn += 1, - Some(StopReason::StopSequence) => self.stop_sequence += 1, - Some(StopReason::ToolUse) => self.tool_use += 1, - Some(StopReason::Refusal) => self.refusal += 1, - Some(StopReason::Silent) => self.silent += 1, - } - } - - /// Fold every turn's `stop_reason` into a fresh counts struct. - pub fn from_turns(turns: &[TurnRecord]) -> Self { - let mut out = Self::default(); - for t in turns { - out.bump(t.stop_reason); - } - out - } - - /// True iff every counter is zero — useful for "skip the outcome line - /// entirely" presentation logic in summary. - pub fn is_empty(&self) -> bool { - self.end_turn - | self.max_tokens - | self.pause_turn - | self.stop_sequence - | self.tool_use - | self.refusal - | self.silent - | self.none - == 0 - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Summary { - pub total_tokens: u64, - pub total_cost: f64, - pub turn_count: u64, - pub by_tool: Vec, - pub by_model: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub by_tag: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub replacement_savings: Option, - /// Per-outcome breakdown — `end_turn` / `max_tokens` / `refusal` / etc. - /// Counts roll up the trailing `stop_reason` of every assistant turn - /// in the filtered slice. See #437. - pub stop_reasons: StopReasonCounts, - /// Count of turns whose model had no entry in the pricing snapshot. - /// Their cost is reported as $0. Zero when all models are priced. - #[serde(default)] - pub unpriced_turns: u64, - /// Distinct model names (first-seen order) that had no pricing entry. - /// Empty when all models are priced. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub unpriced_models: Vec, -} - -impl LedgerHandle { - pub fn summary(&self, opts: SummaryOptions) -> Result { - let mut q = build_query( - opts.session.as_deref(), - opts.project.as_deref(), - opts.since.as_deref(), - )?; - if let Some(tags) = opts.tags.clone() { - validate_tags(&tags)?; - if !tags.is_empty() { - q.enrichment = Some(tags); - } - } - let group_by_tag = opts.group_by_tag.clone(); - if let Some(tag) = group_by_tag.as_deref() { - validate_tag_key(tag, "groupByTag")?; - } - let enriched = self.inner.query_turns(&q)?; - let turns: Vec = enriched.iter().map(|e| e.turn.clone()).collect(); - let pricing = load_pricing(None); - let mut summary = compute_summary(&turns, &pricing); - if let Some(tag) = group_by_tag { - summary.by_tag = Some(compute_summary_by_tag(&enriched, &tag, &pricing)); - } - Ok(summary) - } -} - -pub fn summary(opts: SummaryOptions) -> Result { - let handle = open_with(opts.ledger_home.as_deref())?; - handle.summary(SummaryOptions { - ledger_home: None, - ..opts - }) -} - -fn validate_tags(tags: &Enrichment) -> Result<()> { - for key in tags.keys() { - validate_tag_key(key, "tag")?; - } - Ok(()) -} - -fn validate_tag_key(key: &str, label: &str) -> Result<()> { - if key.is_empty() { - anyhow::bail!("{label} key must be non-empty"); - } - Ok(()) -} - -pub(crate) fn compute_summary(turns: &[TurnRecord], pricing: &PricingTable) -> Summary { - // First-seen iteration order matches TS `Map` semantics. - let mut by_tool_order: Vec = Vec::new(); - let mut by_tool: HashMap = HashMap::new(); - let mut by_model_order: Vec = Vec::new(); - let mut by_model: HashMap = HashMap::new(); - let mut total_tokens: u64 = 0; - let mut total_cost: f64 = 0.0; - - for t in turns { - let cost = cost_for_turn(t, pricing).map(|c| c.total).unwrap_or(0.0); - let tokens = t.usage.input - + t.usage.output - + t.usage.reasoning - + t.usage.cache_read - + t.usage.cache_create_5m - + t.usage.cache_create_1h; - total_tokens += tokens; - total_cost += cost; - - let model_row = by_model.entry(t.model.clone()).or_insert_with(|| { - by_model_order.push(t.model.clone()); - SummaryModelRow { - model: t.model.clone(), - tokens: 0, - cost: 0.0, - } - }); - model_row.tokens += tokens; - model_row.cost += cost; - - for call in &t.tool_calls { - let tool_row = by_tool.entry(call.name.clone()).or_insert_with(|| { - by_tool_order.push(call.name.clone()); - SummaryToolRow { - tool: call.name.clone(), - tokens: 0, - cost: 0.0, - count: 0, - } - }); - tool_row.tokens += tokens; - tool_row.cost += cost; - tool_row.count += 1; - } - } - - let savings = summarize_replacement_savings(turns, None); - let replacement_savings = if savings.calls > 0 { - Some(savings) - } else { - None - }; - - // Use the same pricing table that was used for cost accumulation so the - // count precisely matches which turns contributed $0 to `total_cost`. - let (unpriced_turns, unpriced_models) = tally_unpriced(turns, pricing); - - Summary { - total_tokens, - total_cost, - turn_count: turns.len() as u64, - by_tool: by_tool_order - .into_iter() - .map(|k| by_tool.remove(&k).unwrap()) - .collect(), - by_model: by_model_order - .into_iter() - .map(|k| by_model.remove(&k).unwrap()) - .collect(), - by_tag: None, - replacement_savings, - stop_reasons: StopReasonCounts::from_turns(turns), - unpriced_turns, - unpriced_models, - } -} - -fn compute_summary_by_tag( - enriched: &[EnrichedTurn], - tag: &str, - pricing: &PricingTable, -) -> Vec { - let mut order: Vec> = Vec::new(); - let mut rows: HashMap, SummaryTagRow> = HashMap::new(); - - for e in enriched { - let value = e.enrichment.get(tag).cloned(); - let tokens = total_tokens_for_turn(&e.turn); - let cost = cost_for_turn(&e.turn, pricing) - .map(|c| c.total) - .unwrap_or(0.0); - let row = rows.entry(value.clone()).or_insert_with(|| { - order.push(value.clone()); - SummaryTagRow { - tag: tag.to_string(), - value, - tokens: 0, - cost: 0.0, - turn_count: 0, - } - }); - row.tokens += tokens; - row.cost += cost; - row.turn_count += 1; - } - - let mut out: Vec = order - .into_iter() - .map(|k| rows.remove(&k).unwrap()) - .collect(); - out.sort_by(|a, b| { - b.cost - .partial_cmp(&a.cost) - .unwrap_or(std::cmp::Ordering::Equal) - }); - out -} - -fn total_tokens_for_turn(t: &TurnRecord) -> u64 { - t.usage.input - + t.usage.output - + t.usage.reasoning - + t.usage.cache_read - + t.usage.cache_create_5m - + t.usage.cache_create_1h -} - -// --------------------------------------------------------------------------- -// richer summary report -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryReportOptions { - pub session: Option, - pub project: Option, - pub since: Option, - pub workflow: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tags: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub group_by_tag: Option, - pub agent: Option, - /// Provider labels to keep. Values are trimmed and matched - /// case-insensitively against the SDK's effective provider resolver. - #[serde(default)] - pub providers: Option>, - #[serde(default)] - pub mode: SummaryReportMode, - #[serde(default)] - pub include_quality: bool, - pub ledger_home: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase", tag = "kind")] -pub enum SummaryReportMode { - Grouped { - #[serde(default)] - by_provider: bool, - }, - ByTool, - BySubagentType, - ByRelationship { - #[serde(default)] - subagent: bool, - }, - SubagentTree { - #[serde(default)] - session_id: Option, - }, -} - -impl Default for SummaryReportMode { - fn default() -> Self { - Self::Grouped { by_provider: false } - } -} - -#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -pub enum SummaryGroupBy { - Model, - Provider, - Tag, -} - -impl SummaryGroupBy { - pub fn wire_str(self) -> &'static str { - match self { - Self::Model => "model", - Self::Provider => "provider", - Self::Tag => "tag", - } - } - - pub fn json_key(self) -> &'static str { - match self { - Self::Model => "byModel", - Self::Provider => "byProvider", - Self::Tag => "byTag", - } - } -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -#[allow(clippy::large_enum_variant)] -pub enum SummaryReport { - Grouped(SummaryGroupedReport), - ByTool(SummaryByToolReport), - BySubagentType(SummarySubagentTypeReport), - Relationship(SummaryRelationshipReport), - SubagentTree(SummarySubagentTreeReport), -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryGroupedReport { - pub group_by: SummaryGroupBy, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tag_key: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub tag_values: Vec>, - pub turn_count: u64, - pub rows: Vec, - pub total_cost: CostBreakdown, - pub fidelity: FidelitySummary, - /// Stable TS-compatible JSON shape for per-cell coverage. Kept in the SDK - /// so presenters don't rebuild order-sensitive HashMap projections. - pub per_cell_fidelity: serde_json::Value, - pub replacement_savings: ReplacementSavingsSummary, - /// Per-outcome turn counts (issue #437). Always populated; presenters - /// decide whether to render the line based on `is_empty()`. - pub stop_reasons: StopReasonCounts, - /// Paired / orphan subagent transcript counts (issue #435). Populated - /// by a lazy walk over the Claude `~/.claude/projects/` tree at - /// summary time — when no sidecars exist anywhere reachable the - /// `read_dir` short-circuits and the field stays at - /// `SubagentCounts::default()`. Presenters render the - /// `subagents: X paired, Y orphan` line only when - /// `!subagents.is_empty()`. - #[serde( - default, - skip_serializing_if = "crate::reader::SubagentCounts::is_empty" - )] - pub subagents: crate::reader::SubagentCounts, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub quality: Option, - /// Count of turns whose model had no entry in the pricing snapshot. - /// Their cost is reported as $0. Zero when all models are priced. - #[serde(default)] - pub unpriced_turns: u64, - /// Distinct model names (first-seen order) that had no pricing entry. - /// Empty when all models are priced. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub unpriced_models: Vec, -} - -#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] -#[serde(rename_all = "kebab-case")] -pub enum SummaryToolAttributionMethod { - Unattributed, - Sized, - EvenSplit, -} - -impl SummaryToolAttributionMethod { - pub fn wire_str(self) -> &'static str { - match self { - Self::Unattributed => "unattributed", - Self::Sized => "sized", - Self::EvenSplit => "even-split", - } - } -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryToolAttributionRow { - pub tool: String, - pub calls: u64, - pub attributed_cost: f64, - pub attribution_method: SummaryToolAttributionMethod, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub savings: Option, -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryByToolReport { - pub turn_count: u64, - pub rows: Vec, - pub unattributed_cost: f64, - pub fidelity: FidelitySummary, - pub replacement_savings: ReplacementSavingsSummary, -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SummarySubagentTypeReport { - pub stats: Vec, -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryRelationshipReport { - pub relationships: Vec, - pub subagent_types: Vec, -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryRelationshipStats { - pub relationship_type: RelationshipType, - pub count: u64, - pub session_count: u64, - pub turn_count: u64, - pub total_cost: f64, - pub median_cost: f64, - pub p95_cost: f64, - pub mean_cost: f64, -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryRelationshipSubagentStats { - pub subagent_type: String, - pub invocations: u64, - pub turns: u64, - pub total_cost: f64, - pub median_cost: f64, - pub p95_cost: f64, - pub mean_cost: f64, -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SummarySubagentTreeReport { - pub session_id: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub root: Option, -} - -/// One time bucket of a [`SummaryTimeseries`]: the grouped summary totals for -/// turns whose `ts` falls in `[start, end)`. -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryBucket { - pub start: String, - pub end: String, - pub turn_count: u64, - pub total_tokens: u64, - pub total_cost: CostBreakdown, - pub group_by: SummaryGroupBy, - pub rows: Vec, -} - -/// A time-series of grouped summary totals — one [`SummaryBucket`] per -/// `bucket_secs`-wide window across the `--since` range. Produced by -/// [`LedgerHandle::summary_timeseries`]. -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SummaryTimeseries { - #[serde(rename = "bucketSeconds")] - pub bucket_secs: u64, - pub buckets: Vec, -} - -impl LedgerHandle { - /// Time-bucketed cost/usage totals (the `--bucket` form of the default - /// grouped summary). Fetches the `--since` window once, then partitions the - /// turns by `ts` into `bucket_secs`-wide buckets and aggregates each — a - /// pure per-turn fold, so per-bucket totals sum back to the un-bucketed - /// total. Supported only for the default grouped (`byModel`/`byProvider`) - /// summary; the tool/subagent/relationship attribution modes are rejected. - pub fn summary_timeseries( - &self, - opts: SummaryReportOptions, - bucket_secs: u64, - ) -> Result { - let by_provider = match &opts.mode { - SummaryReportMode::Grouped { by_provider } => *by_provider, - _ => anyhow::bail!( - "--bucket is only supported with the default grouped summary, not \ - --by-tool/--by-subagent-type/--by-relationship/--subagent-tree" - ), - }; - if opts.group_by_tag.is_some() { - anyhow::bail!("--bucket is not supported with --group-by-tag"); - } - if opts.include_quality { - anyhow::bail!("--bucket is not supported with --quality metrics yet"); - } - - let q = build_summary_report_query(&opts)?; - let provider_filter = normalize_summary_provider_filter(opts.providers.as_deref()); - let pricing = load_pricing(None); - let agent_session_ids = match opts.agent.as_deref() { - Some(agent_id) => Some(resolve_summary_agent_session_tree(&self.inner, agent_id)?), - None => None, - }; - - let enriched = self.inner.query_turns(&q)?; - let enriched = filter_summary_enriched_turns( - enriched, - opts.agent.as_deref(), - agent_session_ids.as_ref(), - provider_filter.as_ref(), - ); - let turns = summary_turns_from_enriched(&enriched); - - let Some((buckets, per_bucket)) = - super::partition_into_buckets(turns, q.since.as_deref(), bucket_secs, |t| &t.ts)? - else { - return Ok(SummaryTimeseries { - bucket_secs, - buckets: Vec::new(), - }); - }; - - let group_by = if by_provider { - SummaryGroupBy::Provider - } else { - SummaryGroupBy::Model - }; - let out = per_bucket - .into_iter() - .enumerate() - .map(|(i, bturns)| { - let rows = if by_provider { - aggregate_by_provider(&bturns, AggregateByProviderOptions::new(&pricing)) - .into_iter() - .map(summary_provider_to_aggregate_row) - .collect::>() - } else { - summary_aggregate_by_model(&bturns, &pricing) - }; - let total_cost = sum_costs(rows.iter().map(|r| &r.cost)); - let total_tokens: u64 = bturns.iter().map(total_tokens_for_turn).sum(); - SummaryBucket { - start: buckets.start_iso(i), - end: buckets.end_iso(i), - turn_count: bturns.len() as u64, - total_tokens, - total_cost, - group_by, - rows, - } - }) - .collect(); - - Ok(SummaryTimeseries { - bucket_secs, - buckets: out, - }) - } - - pub fn summary_report(&self, opts: SummaryReportOptions) -> Result { - let q = build_summary_report_query(&opts)?; - let provider_filter = normalize_summary_provider_filter(opts.providers.as_deref()); - let pricing = load_pricing(None); - let agent_session_ids = match opts.agent.as_deref() { - Some(agent_id) => Some(resolve_summary_agent_session_tree(&self.inner, agent_id)?), - None => None, - }; - - if let SummaryReportMode::SubagentTree { session_id } = &opts.mode { - let session_id = session_id - .as_deref() - .filter(|s| !s.is_empty()) - .map(str::to_string) - .or_else(|| q.session_id.clone()) - .ok_or_else(|| anyhow::anyhow!("subagent tree summary requires a session id"))?; - let relationships = - collect_summary_subagent_tree_relationships(&self.inner, &session_id, &q)?; - let enriched = - load_summary_subagent_tree_turns(&self.inner, &session_id, &relationships, &q)?; - let enriched = filter_summary_enriched_turns( - enriched, - opts.agent.as_deref(), - agent_session_ids.as_ref(), - provider_filter.as_ref(), - ); - let turns = summary_turns_from_enriched(&enriched); - let tree_opts = - BuildSubagentTreeOptions::new(&pricing).with_relationships(&relationships); - let trees = build_subagent_tree(&turns, &tree_opts); - let root = trees - .get(&session_id) - .cloned() - .or_else(|| find_summary_tree_node(trees.values(), &session_id)); - return Ok(SummaryReport::SubagentTree(SummarySubagentTreeReport { - session_id, - root, - })); - } - - let enriched = self.inner.query_turns(&q)?; - let enriched = filter_summary_enriched_turns( - enriched, - opts.agent.as_deref(), - agent_session_ids.as_ref(), - provider_filter.as_ref(), - ); - let turns = summary_turns_from_enriched(&enriched); - - match opts.mode { - SummaryReportMode::Grouped { by_provider } => { - let (group_by, tag_key, tag_values, rows) = if let Some(tag_key) = - opts.group_by_tag.as_deref() - { - let (rows, values) = summary_aggregate_by_tag(&enriched, tag_key, &pricing); - (SummaryGroupBy::Tag, Some(tag_key.to_string()), values, rows) - } else if by_provider { - ( - SummaryGroupBy::Provider, - None, - Vec::new(), - aggregate_by_provider(&turns, AggregateByProviderOptions::new(&pricing)) - .into_iter() - .map(summary_provider_to_aggregate_row) - .collect(), - ) - } else { - ( - SummaryGroupBy::Model, - None, - Vec::new(), - summary_aggregate_by_model(&turns, &pricing), - ) - }; - let total_cost = sum_costs(rows.iter().map(|r| &r.cost)); - let fidelity = summarize_fidelity(&turns); - let per_cell_fidelity = summary_per_cell_fidelity_to_value(&rows, group_by); - let replacement_savings = summarize_replacement_savings(&turns, None); - let quality = if opts.include_quality { - Some(compute_summary_quality_for_turns(&self.inner, &turns)?) - } else { - None - }; - let stop_reasons = StopReasonCounts::from_turns(&turns); - // Lazy walk over `~/.claude/projects/` (or the configured - // override) for the `subagents: X paired, Y orphan` - // summary line (issue #435). The walk short-circuits when - // the projects root is missing or every session lacks a - // `subagents/` subdir — i.e. zero cost on the vast - // majority of summaries that don't hit a session with - // sidecar transcripts. - // - // When the summary itself is scoped (any of `--session`, - // `--project`, `--since`, `--workflow`, `--tags`, - // `--agent`, `--providers`) we restrict the sidecar - // walk to the same session-id set the rest of the - // summary covers; otherwise the line could report - // paired/orphan counts from sessions the user excluded. - // Un-filtered runs keep the original global walk - // behavior. - let session_filter = summary_subagent_session_filter(&opts, &turns); - let subagents = compute_summary_subagent_counts(session_filter.as_ref()); - let (unpriced_turns, unpriced_models) = tally_unpriced(&turns, &pricing); - Ok(SummaryReport::Grouped(SummaryGroupedReport { - group_by, - tag_key, - tag_values, - turn_count: turns.len() as u64, - rows, - total_cost, - fidelity, - per_cell_fidelity, - replacement_savings, - stop_reasons, - subagents, - quality, - unpriced_turns, - unpriced_models, - })) - } - SummaryReportMode::ByTool => { - let attribution_turns = - load_summary_by_tool_attribution_turns(&self.inner, &enriched, &q)?; - let report = compute_summary_by_tool_report( - &self.inner, - &turns, - &attribution_turns, - &pricing, - )?; - Ok(SummaryReport::ByTool(report)) - } - SummaryReportMode::BySubagentType => { - let stats = - aggregate_subagent_type_stats(&turns, &BuildSubagentTreeOptions::new(&pricing)); - Ok(SummaryReport::BySubagentType(SummarySubagentTypeReport { - stats, - })) - } - SummaryReportMode::ByRelationship { subagent } => { - let relationships = self - .inner - .query_relationships(&summary_relationship_query_for_turn_slice(&q))?; - let matches = - match_summary_relationships_to_turns(&relationships, &turns, &pricing); - let stats = aggregate_summary_relationship_stats(&matches); - if subagent { - let subagent_types = aggregate_summary_relationship_subagent_stats(&matches); - let relationships = stats - .into_iter() - .filter(|s| s.relationship_type == RelationshipType::Subagent) - .collect(); - Ok(SummaryReport::Relationship(SummaryRelationshipReport { - relationships, - subagent_types, - })) - } else { - Ok(SummaryReport::Relationship(SummaryRelationshipReport { - relationships: stats, - subagent_types: Vec::new(), - })) - } - } - SummaryReportMode::SubagentTree { .. } => unreachable!(), - } - } -} - -pub fn summary_report(opts: SummaryReportOptions) -> Result { - let handle = open_with(opts.ledger_home.as_deref())?; - handle.summary_report(SummaryReportOptions { - ledger_home: None, - ..opts - }) -} - -pub fn summary_fidelity_summary_to_value(s: &FidelitySummary) -> serde_json::Value { - let mut by_class = serde_json::Map::new(); - for class in [ - FidelityClass::Full, - FidelityClass::UsageOnly, - FidelityClass::AggregateOnly, - FidelityClass::CostOnly, - FidelityClass::Partial, - ] { - by_class.insert( - class.wire_str().to_string(), - serde_json::json!(*s.by_class.get(&class).unwrap_or(&0)), - ); - } - - let mut by_granularity = serde_json::Map::new(); - for g in [ - UsageGranularity::PerTurn, - UsageGranularity::PerMessage, - UsageGranularity::PerSessionAggregate, - UsageGranularity::CostOnly, - ] { - by_granularity.insert( - g.wire_str().to_string(), - serde_json::json!(*s.by_granularity.get(&g).unwrap_or(&0)), - ); - } - - let mut missing = serde_json::Map::new(); - for field in [ - "hasInputTokens", - "hasOutputTokens", - "hasReasoningTokens", - "hasCacheReadTokens", - "hasCacheCreateTokens", - "hasToolCalls", - "hasToolResultEvents", - "hasSessionRelationships", - "hasRawContent", - ] { - missing.insert( - field.to_string(), - serde_json::json!(*s.missing_coverage.get(field).unwrap_or(&0)), - ); - } - - let mut out = serde_json::Map::new(); - out.insert("total".into(), serde_json::json!(s.total)); - out.insert("byClass".into(), serde_json::Value::Object(by_class)); - out.insert( - "byGranularity".into(), - serde_json::Value::Object(by_granularity), - ); - out.insert("missingCoverage".into(), serde_json::Value::Object(missing)); - out.insert("unknown".into(), serde_json::json!(s.unknown)); - serde_json::Value::Object(out) -} - -pub fn summary_per_cell_fidelity_to_value( - rows: &[UsageCostAggregateRow], - group_by: SummaryGroupBy, -) -> serde_json::Value { - let cells: Vec = rows - .iter() - .map(|r| { - let fields = [ - ("input", &r.coverage.input), - ("output", &r.coverage.output), - ("reasoning", &r.coverage.reasoning), - ("cacheRead", &r.coverage.cache_read), - ("cacheCreate", &r.coverage.cache_create), - ]; - let mut fields_map = serde_json::Map::new(); - let mut partial = false; - for (name, c) in fields { - if summary_cell_is_partial(c) || (c.known == 0 && c.missing > 0) { - partial = true; - } - fields_map.insert( - name.to_string(), - serde_json::json!({ - "known": c.known, - "missing": c.missing, - }), - ); - } - serde_json::json!({ - "label": r.label, - "partial": partial, - "fields": serde_json::Value::Object(fields_map), - }) - }) - .collect(); - serde_json::json!({ - "groupBy": group_by.wire_str(), - "cells": cells, - }) -} - -pub fn summary_replacement_savings_to_value( - savings: &ReplacementSavingsSummary, -) -> serde_json::Value { - let mut by_tool: Vec = savings - .by_tool - .iter() - .map(|(name, agg)| { - serde_json::json!({ - "tool": name, - "calls": agg.calls, - "collapsedCalls": agg.collapsed_calls, - "estimatedTokensSaved": agg.estimated_tokens_saved, - }) - }) - .collect(); - by_tool.sort_by(|a, b| { - let av = a - .get("estimatedTokensSaved") - .and_then(serde_json::Value::as_u64) - .unwrap_or(0); - let bv = b - .get("estimatedTokensSaved") - .and_then(serde_json::Value::as_u64) - .unwrap_or(0); - bv.cmp(&av).then_with(|| { - let at = a - .get("tool") - .and_then(serde_json::Value::as_str) - .unwrap_or(""); - let bt = b - .get("tool") - .and_then(serde_json::Value::as_str) - .unwrap_or(""); - at.cmp(bt) - }) - }); - serde_json::json!({ - "calls": savings.calls, - "collapsedCalls": savings.collapsed_calls, - "estimatedTokensSaved": savings.estimated_tokens_saved, - "byTool": by_tool, - }) -} - -fn build_summary_report_query(opts: &SummaryReportOptions) -> Result { - let mut q = build_query( - opts.session.as_deref(), - opts.project.as_deref(), - opts.since.as_deref(), - )?; - if let Some(tag) = opts.group_by_tag.as_deref() { - validate_tag_key(tag, "groupByTag")?; - } - let mut enrichment = BTreeMap::new(); - if let Some(workflow) = &opts.workflow { - enrichment.insert("workflowId".to_string(), workflow.clone()); - } - if let Some(tags) = opts.tags.as_ref() { - validate_tags(tags)?; - for (key, value) in tags { - if let Some(existing) = enrichment.get(key) { - if existing != value { - anyhow::bail!( - "conflicting filters for tag \"{key}\" ({existing:?} vs {value:?})" - ); - } - } - enrichment.insert(key.clone(), value.clone()); - } - } - if !enrichment.is_empty() { - q.enrichment = Some(enrichment); - } - Ok(q) -} - -fn normalize_summary_provider_filter(providers: Option<&[String]>) -> Option { - let providers: ProviderFilter = providers - .unwrap_or(&[]) - .iter() - .map(|s| s.trim().to_ascii_lowercase()) - .filter(|s| !s.is_empty()) - .collect(); - if providers.is_empty() { - None - } else { - Some(providers) - } -} - -fn filter_summary_enriched_turns( - turns: Vec, - agent_id: Option<&str>, - agent_session_ids: Option<&HashSet>, - provider_filter: Option<&ProviderFilter>, -) -> Vec { - turns - .into_iter() - .filter(|t| summary_agent_passes(t, agent_id, agent_session_ids)) - .filter(|t| summary_provider_passes(&t.turn, provider_filter)) - .collect() -} - -fn summary_agent_passes( - t: &EnrichedTurn, - agent_id: Option<&str>, - session_ids: Option<&HashSet>, -) -> bool { - let Some(agent_id) = agent_id else { - return true; - }; - if t.enrichment.get("agentId").map(String::as_str) == Some(agent_id) { - return true; - } - if t.enrichment.get("parentAgentId").map(String::as_str) == Some(agent_id) { - return true; - } - session_ids - .map(|ids| ids.contains(&t.turn.session_id)) - .unwrap_or(false) -} - -fn summary_provider_passes(t: &TurnRecord, provider_filter: Option<&ProviderFilter>) -> bool { - let Some(filter) = provider_filter else { - return true; - }; - let provider = provider_for(t).provider.to_ascii_lowercase(); - filter.contains(&provider) -} - -fn summary_turns_from_enriched(enriched: &[EnrichedTurn]) -> Vec { - enriched.iter().map(|e| e.turn.clone()).collect() -} - -fn load_summary_by_tool_attribution_turns( - ledger: &crate::ledger::Ledger, - selected: &[EnrichedTurn], - q: &Query, -) -> Result> { - let session_ids: Vec = selected - .iter() - .map(|e| e.turn.session_id.clone()) - .collect::>() - .into_iter() - .collect(); - let turns = ledger.query_turns_in_sessions( - &Query { - source: q.source, - ..Default::default() - }, - &session_ids, - )?; - let mut by_key: IndexMap = IndexMap::new(); - for t in turns { - let key = format!( - "{}|{}|{}", - t.turn.source.wire_str(), - t.turn.session_id, - t.turn.message_id, - ); - by_key.insert(key, t); - } - Ok(by_key.into_values().map(|e| e.turn).collect()) -} - -fn resolve_summary_agent_session_tree( - ledger: &crate::ledger::Ledger, - agent_id: &str, -) -> Result> { - Ok(collect_summary_agent_session_tree( - &ledger.query_relationships(&Query::default())?, - agent_id, - )) -} - -pub(crate) fn collect_summary_agent_session_tree( - relationships: &[SessionRelationshipRecord], - agent_id: &str, -) -> HashSet { - let mut by_parent: HashMap> = HashMap::new(); - for r in relationships { - if r.relationship_type != RelationshipType::Subagent { - continue; - } - let Some(parent) = r.related_session_id.as_deref() else { - continue; - }; - if parent.is_empty() { - continue; - } - by_parent.entry(parent.to_string()).or_default().push(r); - } - - let mut sessions = HashSet::new(); - let mut seen = HashSet::new(); - let mut queue = VecDeque::from([agent_id.to_string()]); - while let Some(parent) = queue.pop_front() { - if !seen.insert(parent.clone()) { - continue; - } - for child in by_parent.get(&parent).into_iter().flatten() { - sessions.insert(child.session_id.clone()); - queue.push_back(child.session_id.clone()); - if let Some(agent) = child.agent_id.as_ref() { - if !agent.is_empty() { - queue.push_back(agent.clone()); - } - } - } - } - sessions -} - -/// Resolve the Claude projects root and run [`count_subagents_under`] -/// against it for the `subagents: X paired, Y orphan` summary line. -/// -/// We honor `BURN_CLAUDE_PROJECTS_DIR` so tests (and integration -/// fixtures) can point at a sandbox without scanning the developer's -/// `~/.claude`. The env var also lets the CLI summary remain -/// reproducible against a fixture-only test suite. When unset we fall -/// back to `$HOME/.claude/projects`; if that doesn't exist the -/// underlying walk returns `(0, 0)` and the summary line is skipped. -/// -/// `session_filter` matches the rest of the summary's filter set: -/// `None` means "no filter — count every session reachable from the -/// projects root" (the un-filtered `burn summary` path); `Some(set)` -/// means "only count sidecars whose session id is in `set`" so a -/// `burn summary --session A` / `--project B` / `--since 24h` run gets -/// a subagent count scoped to the same sessions the rest of the -/// numbers cover. -fn compute_summary_subagent_counts( - session_filter: Option<&HashSet>, -) -> crate::reader::SubagentCounts { - use crate::reader::count_subagents_under; - let root = if let Some(p) = std::env::var_os("BURN_CLAUDE_PROJECTS_DIR") { - std::path::PathBuf::from(p) - } else { - // Defaults to `~/.claude/projects` (HOME, then USERPROFILE on - // Windows — see crate::util::home_dir) when - // `BURN_CLAUDE_PROJECTS_DIR` is unset. - crate::util::home_dir().join(".claude").join("projects") - }; - count_subagents_under(&root, session_filter) -} - -/// Build the session-id filter set the subagent counter should descend -/// into. Returns `None` when `opts` carries no scoping filters, which -/// preserves the original "scan every reachable session" behavior for -/// the bare `burn summary` invocation. Returns `Some(set)` when any -/// filter (`session`, `project`, `since`, `workflow`, `tags`, `agent`, -/// `providers`) is active — `set` is the session ids that survived -/// every filter, derived from the already-filtered `turns` slice. -/// -/// Plumbing the filter via the filtered turn set (instead of e.g. -/// duplicating the SQL filters inside the walker) ensures the count -/// can never diverge from the rest of the summary numbers: anything -/// that drops a session from the row aggregates also drops it from the -/// subagent count. -pub(crate) fn summary_subagent_session_filter( - opts: &SummaryReportOptions, - turns: &[TurnRecord], -) -> Option> { - let has_filter = opts.session.is_some() - || opts.project.is_some() - || opts.since.is_some() - || opts.workflow.is_some() - || opts.agent.is_some() - || opts.tags.as_ref().map(|t| !t.is_empty()).unwrap_or(false) - || opts - .providers - .as_ref() - .map(|p| !p.is_empty()) - .unwrap_or(false); - if !has_filter { - return None; - } - Some(turns.iter().map(|t| t.session_id.clone()).collect()) -} - -fn compute_summary_quality_for_turns( - ledger: &crate::ledger::Ledger, - turns: &[TurnRecord], -) -> Result { - let content_by_session = load_summary_content_for_quality(ledger, turns)?; - Ok(compute_quality( - turns, - &ComputeQualityOptions { - content_by_session: Some(&content_by_session), - now_ms: None, - }, - )) -} - -fn load_summary_content_for_quality( - ledger: &crate::ledger::Ledger, - turns: &[TurnRecord], -) -> Result>> { - let mut seen = HashSet::new(); - let mut out = HashMap::new(); - for t in turns { - if !seen.insert(t.session_id.clone()) { - continue; - } - let records = ledger.query_content(&Query { - session_id: Some(t.session_id.clone()), - ..Default::default() - })?; - if !records.is_empty() { - out.insert(t.session_id.clone(), records); - } - } - Ok(out) -} - -fn summary_aggregate_by_tag( - enriched: &[EnrichedTurn], - tag_key: &str, - pricing: &PricingTable, -) -> (Vec, Vec>) { - let mut by_value: HashMap, UsageCostAggregateRow> = HashMap::new(); - let mut order: Vec> = Vec::new(); - for enriched in enriched { - let value = enriched.enrichment.get(tag_key).cloned(); - let label = value.clone().unwrap_or_else(|| "(untagged)".to_string()); - let row = by_value.entry(value.clone()).or_insert_with(|| { - order.push(value.clone()); - summary_empty_row(&label) - }); - row.turns += 1; - row.usage.input += enriched.turn.usage.input; - row.usage.output += enriched.turn.usage.output; - row.usage.reasoning += enriched.turn.usage.reasoning; - row.usage.cache_read += enriched.turn.usage.cache_read; - row.usage.cache_create_5m += enriched.turn.usage.cache_create_5m; - row.usage.cache_create_1h += enriched.turn.usage.cache_create_1h; - summary_accumulate_coverage( - &mut row.coverage, - enriched.turn.fidelity.as_ref().map(|f| &f.coverage), - ); - if let Some(c) = cost_for_turn(&enriched.turn, pricing) { - row.cost.total += c.total; - row.cost.input += c.input; - row.cost.output += c.output; - row.cost.reasoning += c.reasoning; - row.cost.cache_read += c.cache_read; - row.cost.cache_create += c.cache_create; - } - } - - let mut pairs: Vec<(Option, UsageCostAggregateRow)> = order - .into_iter() - .map(|value| { - let row = by_value.remove(&value).unwrap(); - (value, row) - }) - .collect(); - pairs.sort_by(|a, b| { - b.1.cost - .total - .partial_cmp(&a.1.cost.total) - .unwrap_or(std::cmp::Ordering::Equal) - }); - let (values, rows): (Vec>, Vec) = - pairs.into_iter().unzip(); - (rows, values) -} - -fn summary_aggregate_by_model( - turns: &[TurnRecord], - pricing: &PricingTable, -) -> Vec { - let mut by_model: IndexMap = IndexMap::new(); - for t in turns { - let key = if t.model.is_empty() { - "unknown".to_string() - } else { - t.model.clone() - }; - let row = by_model - .entry(key.clone()) - .or_insert_with(|| summary_empty_row(&key)); - row.turns += 1; - row.usage.input += t.usage.input; - row.usage.output += t.usage.output; - row.usage.reasoning += t.usage.reasoning; - row.usage.cache_read += t.usage.cache_read; - row.usage.cache_create_5m += t.usage.cache_create_5m; - row.usage.cache_create_1h += t.usage.cache_create_1h; - summary_accumulate_coverage(&mut row.coverage, t.fidelity.as_ref().map(|f| &f.coverage)); - if let Some(c) = cost_for_turn(t, pricing) { - row.cost.total += c.total; - row.cost.input += c.input; - row.cost.output += c.output; - row.cost.reasoning += c.reasoning; - row.cost.cache_read += c.cache_read; - row.cost.cache_create += c.cache_create; - } - } - let mut rows: Vec = by_model.into_values().collect(); - rows.sort_by(|a, b| { - b.cost - .total - .partial_cmp(&a.cost.total) - .unwrap_or(std::cmp::Ordering::Equal) - }); - rows -} - -fn summary_provider_to_aggregate_row(p: ProviderAggregateRow) -> UsageCostAggregateRow { - UsageCostAggregateRow { - label: p.label, - turns: p.turns, - usage: p.usage, - cost: p.cost, - coverage: p.coverage, - } -} - -fn summary_empty_row(label: &str) -> UsageCostAggregateRow { - UsageCostAggregateRow { - label: label.to_string(), - turns: 0, - usage: Usage::default(), - cost: CostBreakdown { - model: label.to_string().into(), - total: 0.0, - input: 0.0, - output: 0.0, - reasoning: 0.0, - cache_read: 0.0, - cache_create: 0.0, - }, - coverage: RowCoverage::default(), - } -} - -fn summary_accumulate_coverage(target: &mut RowCoverage, coverage: Option<&Coverage>) { - for f in [ - CoverageField::Input, - CoverageField::Output, - CoverageField::Reasoning, - CoverageField::CacheRead, - CoverageField::CacheCreate, - ] { - let known = match coverage { - None => true, - Some(c) => match f { - CoverageField::Input => c.has_input_tokens, - CoverageField::Output => c.has_output_tokens, - CoverageField::Reasoning => c.has_reasoning_tokens, - CoverageField::CacheRead => c.has_cache_read_tokens, - CoverageField::CacheCreate => c.has_cache_create_tokens, - }, - }; - let slot = target.field_mut(f); - if known { - slot.known += 1; - } else { - slot.missing += 1; - } - } -} - -fn summary_cell_is_partial(c: &FieldCoverage) -> bool { - c.known > 0 && c.missing > 0 -} - -#[derive(Debug, Default, Clone)] -pub(crate) struct SummaryToolAgg { - pub(crate) calls: u64, - pub(crate) cost: f64, - pub(crate) sized_cost: f64, - pub(crate) even_split_cost: f64, -} - -#[derive(Debug, Default)] -struct SummaryUserTurnSizeBucket { - tool_bytes_by_id: HashMap, - total_bytes: u64, -} - -fn compute_summary_by_tool_report( - ledger: &crate::ledger::Ledger, - turns: &[TurnRecord], - attribution_turns: &[TurnRecord], - pricing: &PricingTable, -) -> Result { - let user_turns_by_session = load_summary_user_turns_for_by_tool(ledger, attribution_turns)?; - let selected_turns = selected_summary_turn_keys(turns); - let (by_tool, unattributed_cost) = attribute_summary_cost_to_tools( - attribution_turns, - pricing, - &user_turns_by_session, - Some(&selected_turns), - ); - let fidelity = summarize_fidelity(turns); - let replacement_savings = summarize_replacement_savings(turns, None); - let mut sorted: Vec<(String, SummaryToolAgg)> = by_tool.into_iter().collect(); - sorted.sort_by(|a, b| { - b.1.cost - .partial_cmp(&a.1.cost) - .unwrap_or(std::cmp::Ordering::Equal) - }); - let rows = sorted - .into_iter() - .map(|(tool, agg)| SummaryToolAttributionRow { - savings: replacement_savings.by_tool.get(&tool).cloned(), - tool, - calls: agg.calls, - attributed_cost: agg.cost, - attribution_method: summary_tool_attribution_method(&agg), - }) - .collect(); - Ok(SummaryByToolReport { - turn_count: turns.len() as u64, - rows, - unattributed_cost, - fidelity, - replacement_savings, - }) -} - -fn load_summary_user_turns_for_by_tool( - ledger: &crate::ledger::Ledger, - turns: &[TurnRecord], -) -> Result>> { - let session_ids: BTreeSet = turns.iter().map(|t| t.session_id.clone()).collect(); - let mut out = HashMap::new(); - for session_id in session_ids { - let rows = ledger.query_user_turns(&Query { - session_id: Some(session_id.clone()), - ..Default::default() - })?; - if !rows.is_empty() { - out.insert(session_id, rows); - } - } - Ok(out) -} - -fn selected_summary_turn_keys(turns: &[TurnRecord]) -> HashSet { - turns.iter().map(summary_turn_identity_key).collect() -} - -pub(crate) fn attribute_summary_cost_to_tools( - turns: &[TurnRecord], - pricing: &PricingTable, - user_turns_by_session: &HashMap>, - selected_turns: Option<&HashSet>, -) -> (IndexMap, f64) { - let mut by_tool: IndexMap = IndexMap::new(); - let mut unattributed = 0.0; - let mut by_session: IndexMap> = IndexMap::new(); - for t in turns { - by_session.entry(t.session_id.clone()).or_default().push(t); - } - - for (session_id, mut list) in by_session { - list.sort_by_key(|t| t.turn_index); - let user_turn_size_index = index_summary_user_turn_block_sizes( - user_turns_by_session - .get(&session_id) - .map(Vec::as_slice) - .unwrap_or(&[]), - ); - for i in 0..list.len() { - let turn = list[i]; - if !summary_turn_is_selected(turn, selected_turns) { - continue; - } - let Some(c) = cost_for_turn(turn, pricing) else { - continue; - }; - let ingest_cost = c.input + c.cache_read + c.cache_create; - - if i == 0 { - unattributed += ingest_cost; - continue; - } - let prior = list[i - 1]; - if prior.tool_calls.is_empty() { - unattributed += ingest_cost; - continue; - } - - let key = summary_bridge_key(&prior.message_id, &turn.message_id); - let sizes = user_turn_size_index.get(&key); - let sized_bytes: u64 = match sizes { - Some(s) => prior - .tool_calls - .iter() - .map(|tc| *s.tool_bytes_by_id.get(&tc.id).unwrap_or(&0)) - .sum(), - None => 0, - }; - if let Some(sizes) = sizes.filter(|_| sized_bytes > 0) { - let allocatable_cost = if sizes.total_bytes > 0 { - ingest_cost * (sized_bytes as f64 / sizes.total_bytes as f64).min(1.0) - } else { - ingest_cost - }; - unattributed += ingest_cost - allocatable_cost; - let mut raw_shares: Vec<(String, f64)> = Vec::new(); - for tc in &prior.tool_calls { - let bytes = *sizes.tool_bytes_by_id.get(&tc.id).unwrap_or(&0); - if bytes == 0 { - continue; - } - by_tool.entry(tc.name.clone()).or_default().calls += 1; - raw_shares.push(( - tc.name.clone(), - (bytes as f64 / sized_bytes as f64) * allocatable_cost, - )); - } - let raw_subtotal: f64 = raw_shares.iter().map(|(_, cost)| *cost).sum(); - let scale = if raw_subtotal > allocatable_cost && raw_subtotal > 0.0 { - allocatable_cost / raw_subtotal - } else { - 1.0 - }; - for (tool, cost) in raw_shares { - let share = cost * scale; - let agg = by_tool.entry(tool).or_default(); - agg.cost += share; - agg.sized_cost += share; - } - } else { - let share = ingest_cost / prior.tool_calls.len() as f64; - for tc in &prior.tool_calls { - let agg = by_tool.entry(tc.name.clone()).or_default(); - agg.calls += 1; - agg.cost += share; - agg.even_split_cost += share; - } - } - } - } - - (by_tool, unattributed) -} - -fn summary_turn_is_selected(turn: &TurnRecord, selected_turns: Option<&HashSet>) -> bool { - selected_turns - .map(|keys| keys.contains(&summary_turn_identity_key(turn))) - .unwrap_or(true) -} - -pub(crate) fn summary_turn_identity_key(turn: &TurnRecord) -> String { - format!( - "{}\0{}\0{}", - turn.source.wire_str(), - turn.session_id, - turn.message_id - ) -} - -fn index_summary_user_turn_block_sizes( - user_turns: &[UserTurnRecord], -) -> HashMap { - let mut out: HashMap = HashMap::new(); - for user_turn in user_turns { - let (Some(preceding), Some(following)) = ( - user_turn.preceding_message_id.as_ref(), - user_turn.following_message_id.as_ref(), - ) else { - continue; - }; - let bucket = out - .entry(summary_bridge_key(preceding, following)) - .or_default(); - for block in &user_turn.blocks { - let bytes = block.byte_len; - bucket.total_bytes += bytes; - if block.kind != UserTurnBlockKind::ToolResult { - continue; - } - let Some(tool_use_id) = block.tool_use_id.as_ref() else { - continue; - }; - *bucket - .tool_bytes_by_id - .entry(tool_use_id.clone()) - .or_default() += bytes; - } - } - out -} - -fn summary_bridge_key(preceding_message_id: &str, following_message_id: &str) -> String { - format!("{preceding_message_id}\0{following_message_id}") -} - -pub(crate) fn summary_tool_attribution_method( - agg: &SummaryToolAgg, -) -> SummaryToolAttributionMethod { - if agg.sized_cost == 0.0 && agg.even_split_cost == 0.0 { - SummaryToolAttributionMethod::Unattributed - } else if agg.sized_cost >= agg.even_split_cost { - SummaryToolAttributionMethod::Sized - } else { - SummaryToolAttributionMethod::EvenSplit - } -} - -const SUMMARY_RELATIONSHIP_ORDER: [RelationshipType; 4] = [ - RelationshipType::Root, - RelationshipType::Continuation, - RelationshipType::Fork, - RelationshipType::Subagent, -]; - -#[derive(Debug, Clone)] -pub(crate) struct SummaryRelationshipMatch { - pub(crate) relationship_type: RelationshipType, - pub(crate) session_id: String, - pub(crate) subagent_type: Option, - pub(crate) turn_count: u64, - pub(crate) cost: f64, -} - -struct SummaryRelationshipTurnIndex<'a> { - all_by_session: HashMap>, - main_by_session: HashMap>, - sidechain_by_session: HashMap>, - subagent_by_session_agent: HashMap>, -} - -fn summary_relationship_query_for_turn_slice(q: &Query) -> Query { - Query { - session_id: q.session_id.clone(), - source: q.source, - ..Default::default() - } -} - -fn match_summary_relationships_to_turns( - relationships: &[SessionRelationshipRecord], - turns: &[TurnRecord], - pricing: &PricingTable, -) -> Vec { - let index = build_summary_relationship_turn_index(turns); - let mut out = Vec::new(); - let mut seen = HashSet::new(); - for r in relationships { - let key = summary_relationship_instance_key(r); - if !seen.insert(key) { - continue; - } - let matched_turns = summary_turns_for_relationship(r, &index); - if matched_turns.is_empty() { - continue; - } - let cost = matched_turns - .iter() - .map(|t| cost_for_turn(t, pricing).map(|c| c.total).unwrap_or(0.0)) - .sum(); - out.push(SummaryRelationshipMatch { - relationship_type: r.relationship_type, - session_id: r.session_id.clone(), - subagent_type: summary_relationship_subagent_type(r, &matched_turns), - turn_count: matched_turns.len() as u64, - cost, - }); - } - out -} - -fn build_summary_relationship_turn_index(turns: &[TurnRecord]) -> SummaryRelationshipTurnIndex<'_> { - let mut index = SummaryRelationshipTurnIndex { - all_by_session: HashMap::new(), - main_by_session: HashMap::new(), - sidechain_by_session: HashMap::new(), - subagent_by_session_agent: HashMap::new(), - }; - for turn in turns { - index - .all_by_session - .entry(turn.session_id.clone()) - .or_default() - .push(turn); - if summary_is_main_thread_turn(turn) { - index - .main_by_session - .entry(turn.session_id.clone()) - .or_default() - .push(turn); - } - if turn - .subagent - .as_ref() - .map(|s| s.is_sidechain) - .unwrap_or(false) - { - index - .sidechain_by_session - .entry(turn.session_id.clone()) - .or_default() - .push(turn); - } - if let Some(agent_id) = turn.subagent.as_ref().and_then(|s| s.agent_id.as_ref()) { - if !agent_id.is_empty() { - index - .subagent_by_session_agent - .entry(summary_session_agent_key(&turn.session_id, agent_id)) - .or_default() - .push(turn); - } - } - } - index -} - -fn summary_turns_for_relationship<'a>( - r: &SessionRelationshipRecord, - index: &'a SummaryRelationshipTurnIndex<'a>, -) -> Vec<&'a TurnRecord> { - match r.relationship_type { - RelationshipType::Root => index - .main_by_session - .get(&r.session_id) - .cloned() - .unwrap_or_default(), - RelationshipType::Subagent => { - if let Some(agent_id) = r.agent_id.as_ref().filter(|s| !s.is_empty()) { - let key = summary_session_agent_key(&r.session_id, agent_id); - if let Some(direct) = index.subagent_by_session_agent.get(&key) { - if !direct.is_empty() { - return direct.clone(); - } - } - if r.session_id == *agent_id { - return index - .all_by_session - .get(&r.session_id) - .cloned() - .unwrap_or_default(); - } - } - if let Some(sidechain) = index.sidechain_by_session.get(&r.session_id) { - if !sidechain.is_empty() { - return sidechain.clone(); - } - } - if r.source.wire_str() == "spawn-env" { - return index - .all_by_session - .get(&r.session_id) - .cloned() - .unwrap_or_default(); - } - Vec::new() - } - RelationshipType::Continuation | RelationshipType::Fork => index - .all_by_session - .get(&r.session_id) - .cloned() - .unwrap_or_default(), - } -} - -pub(crate) fn aggregate_summary_relationship_stats( - matches: &[SummaryRelationshipMatch], -) -> Vec { - #[derive(Default)] - struct RelationshipSessionRollup { - relationship_count: u64, - turn_count: u64, - cost: f64, - } - - let mut by_type: HashMap> = - HashMap::new(); - for m in matches { - let by_session = by_type.entry(m.relationship_type).or_default(); - let current = by_session.entry(m.session_id.clone()).or_default(); - current.relationship_count += 1; - current.turn_count += m.turn_count; - current.cost += m.cost; - } - - let mut out = Vec::new(); - for relationship_type in SUMMARY_RELATIONSHIP_ORDER { - let Some(by_session) = by_type.get(&relationship_type) else { - continue; - }; - if by_session.is_empty() { - continue; - } - let mut costs: Vec = by_session.values().map(|rollup| rollup.cost).collect(); - costs.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - let total_cost: f64 = costs.iter().sum(); - let session_count = by_session.len() as u64; - out.push(SummaryRelationshipStats { - relationship_type, - count: by_session - .values() - .map(|rollup| rollup.relationship_count) - .sum(), - session_count, - turn_count: by_session.values().map(|rollup| rollup.turn_count).sum(), - total_cost, - median_cost: summary_percentile(&costs, 0.5), - p95_cost: summary_percentile(&costs, 0.95), - mean_cost: if session_count > 0 { - total_cost / session_count as f64 - } else { - 0.0 - }, - }); - } - out -} - -fn aggregate_summary_relationship_subagent_stats( - matches: &[SummaryRelationshipMatch], -) -> Vec { - struct Agg { - turns: u64, - total: f64, - costs: Vec, - } - let mut by_type: IndexMap = IndexMap::new(); - for m in matches { - if m.relationship_type != RelationshipType::Subagent { - continue; - } - let ty = m - .subagent_type - .clone() - .unwrap_or_else(|| "(unknown)".to_string()); - let agg = by_type.entry(ty).or_insert_with(|| Agg { - turns: 0, - total: 0.0, - costs: Vec::new(), - }); - agg.turns += m.turn_count; - agg.total += m.cost; - agg.costs.push(m.cost); - } - - let mut out = Vec::new(); - for (subagent_type, mut agg) in by_type { - agg.costs - .sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - let invocations = agg.costs.len() as u64; - out.push(SummaryRelationshipSubagentStats { - subagent_type, - invocations, - turns: agg.turns, - total_cost: agg.total, - median_cost: summary_percentile(&agg.costs, 0.5), - p95_cost: summary_percentile(&agg.costs, 0.95), - mean_cost: if invocations > 0 { - agg.total / invocations as f64 - } else { - 0.0 - }, - }); - } - out.sort_by(|a, b| { - b.total_cost - .partial_cmp(&a.total_cost) - .unwrap_or(std::cmp::Ordering::Equal) - }); - out -} - -fn summary_relationship_subagent_type( - relationship: &SessionRelationshipRecord, - turns: &[&TurnRecord], -) -> Option { - if let Some(st) = &relationship.subagent_type { - return Some(st.clone()); - } - turns.iter().find_map(|t| { - t.subagent - .as_ref() - .and_then(|s| s.subagent_type.as_ref()) - .cloned() - }) -} - -fn summary_relationship_instance_key(r: &SessionRelationshipRecord) -> String { - [ - r.source.wire_str(), - r.relationship_type.wire_str(), - &r.session_id, - r.related_session_id.as_deref().unwrap_or(""), - r.agent_id.as_deref().unwrap_or(""), - r.parent_tool_use_id.as_deref().unwrap_or(""), - ] - .join("\0") -} - -fn summary_session_agent_key(session_id: &str, agent_id: &str) -> String { - format!("{session_id}\0{agent_id}") -} - -fn summary_is_main_thread_turn(turn: &TurnRecord) -> bool { - match &turn.subagent { - None => true, - Some(sub) => !sub.is_sidechain || sub.agent_id.as_deref() == Some(&turn.session_id), - } -} - -fn summary_percentile(sorted: &[f64], p: f64) -> f64 { - if sorted.is_empty() { - return 0.0; - } - let rank = - ((p * sorted.len() as f64).ceil() as i64 - 1).clamp(0, sorted.len() as i64 - 1) as usize; - sorted[rank] -} - -fn collect_summary_subagent_tree_relationships( - ledger: &crate::ledger::Ledger, - session_id: &str, - q: &Query, -) -> Result> { - let relationships = ledger.query_relationships(&Query { - source: q.source, - ..Default::default() - })?; - Ok(collect_summary_connected_relationships( - &relationships, - session_id, - )) -} - -pub(crate) fn collect_summary_connected_relationships( - relationships: &[SessionRelationshipRecord], - session_id: &str, -) -> Vec { - let mut by_id: HashMap> = HashMap::new(); - for (idx, r) in relationships.iter().enumerate() { - for id in summary_relationship_connected_ids(r) { - if !id.is_empty() { - by_id.entry(id).or_default().push(idx); - } - } - } - - let mut out: IndexMap = IndexMap::new(); - let mut seen_ids = HashSet::new(); - let mut queue = VecDeque::from([session_id.to_string()]); - while let Some(id) = queue.pop_front() { - if !seen_ids.insert(id.clone()) { - continue; - } - let Some(rows) = by_id.get(&id) else { - continue; - }; - for idx in rows { - let r = &relationships[*idx]; - for next in summary_relationship_connected_ids(r) { - if !next.is_empty() && !seen_ids.contains(&next) { - queue.push_back(next); - } - } - out.insert(summary_relationship_instance_key(r), r.clone()); - } - } - out.into_values().collect() -} - -fn summary_relationship_connected_ids(r: &SessionRelationshipRecord) -> Vec { - let mut ids = vec![r.session_id.clone()]; - if let Some(related) = &r.related_session_id { - ids.push(related.clone()); - } - if let Some(agent) = &r.agent_id { - ids.push(agent.clone()); - } - ids -} - -fn load_summary_subagent_tree_turns( - ledger: &crate::ledger::Ledger, - session_id: &str, - relationships: &[SessionRelationshipRecord], - q: &Query, -) -> Result> { - let mut session_ids = HashSet::from([session_id.to_string()]); - for r in relationships { - session_ids.insert(r.session_id.clone()); - } - - let mut by_key: IndexMap = IndexMap::new(); - for id in session_ids { - let turns = ledger.query_turns(&Query { - session_id: Some(id), - ..q.clone() - })?; - for t in turns { - let key = format!( - "{}|{}|{}", - t.turn.source.wire_str(), - t.turn.session_id, - t.turn.message_id, - ); - by_key.insert(key, t); - } - } - Ok(by_key.into_values().collect()) -} - -fn find_summary_tree_node<'a>( - trees: impl IntoIterator, - node_id: &str, -) -> Option { - for root in trees { - if let Some(found) = find_summary_node(root, node_id) { - return Some(found.clone()); - } - } - None -} - -fn find_summary_node<'a>( - node: &'a SubagentTreeNode, - node_id: &str, -) -> Option<&'a SubagentTreeNode> { - if node.node_id == node_id { - return Some(node); - } - for child in &node.children { - if let Some(found) = find_summary_node(child, node_id) { - return Some(found); - } - } - None -} diff --git a/crates/relayburn-sdk/src/query_verbs/summary/compute.rs b/crates/relayburn-sdk/src/query_verbs/summary/compute.rs new file mode 100644 index 00000000..0d174201 --- /dev/null +++ b/crates/relayburn-sdk/src/query_verbs/summary/compute.rs @@ -0,0 +1,1125 @@ +//! Private compute engine for the `summary` / `summary_report` verbs. +//! +//! This module holds the query-building and aggregation helpers that the +//! `LedgerHandle::summary_report` / `summary_timeseries` dispatchers in +//! `super` drive. The public option/report types and the dispatchers +//! themselves live in `super` (`query_verbs/summary/mod.rs`). + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; + +use anyhow::Result; +use indexmap::IndexMap; + +use crate::analyze::{ + compute_quality, cost_for_turn, provider_for, summarize_fidelity, + summarize_replacement_savings, ComputeQualityOptions, CostBreakdown, CoverageField, + FieldCoverage, PricingTable, ProviderAggregateRow, ProviderFilter, QualityResult, RowCoverage, + SubagentTreeNode, UsageCostAggregateRow, +}; +use crate::ledger::{EnrichedTurn, Query}; +use crate::reader::{ + ContentRecord, Coverage, RelationshipType, SessionRelationshipRecord, TurnRecord, Usage, + UserTurnBlockKind, UserTurnRecord, +}; + +use super::super::build_query; +use super::*; + +pub(crate) fn build_summary_report_query(opts: &SummaryReportOptions) -> Result { + let mut q = build_query( + opts.session.as_deref(), + opts.project.as_deref(), + opts.since.as_deref(), + )?; + if let Some(tag) = opts.group_by_tag.as_deref() { + validate_tag_key(tag, "groupByTag")?; + } + let mut enrichment = BTreeMap::new(); + if let Some(workflow) = &opts.workflow { + enrichment.insert("workflowId".to_string(), workflow.clone()); + } + if let Some(tags) = opts.tags.as_ref() { + validate_tags(tags)?; + for (key, value) in tags { + if let Some(existing) = enrichment.get(key) { + if existing != value { + anyhow::bail!( + "conflicting filters for tag \"{key}\" ({existing:?} vs {value:?})" + ); + } + } + enrichment.insert(key.clone(), value.clone()); + } + } + if !enrichment.is_empty() { + q.enrichment = Some(enrichment); + } + Ok(q) +} + +pub(crate) fn normalize_summary_provider_filter( + providers: Option<&[String]>, +) -> Option { + let providers: ProviderFilter = providers + .unwrap_or(&[]) + .iter() + .map(|s| s.trim().to_ascii_lowercase()) + .filter(|s| !s.is_empty()) + .collect(); + if providers.is_empty() { + None + } else { + Some(providers) + } +} + +pub(crate) fn filter_summary_enriched_turns( + turns: Vec, + agent_id: Option<&str>, + agent_session_ids: Option<&HashSet>, + provider_filter: Option<&ProviderFilter>, +) -> Vec { + turns + .into_iter() + .filter(|t| summary_agent_passes(t, agent_id, agent_session_ids)) + .filter(|t| summary_provider_passes(&t.turn, provider_filter)) + .collect() +} + +pub(crate) fn summary_agent_passes( + t: &EnrichedTurn, + agent_id: Option<&str>, + session_ids: Option<&HashSet>, +) -> bool { + let Some(agent_id) = agent_id else { + return true; + }; + if t.enrichment.get("agentId").map(String::as_str) == Some(agent_id) { + return true; + } + if t.enrichment.get("parentAgentId").map(String::as_str) == Some(agent_id) { + return true; + } + session_ids + .map(|ids| ids.contains(&t.turn.session_id)) + .unwrap_or(false) +} + +pub(crate) fn summary_provider_passes( + t: &TurnRecord, + provider_filter: Option<&ProviderFilter>, +) -> bool { + let Some(filter) = provider_filter else { + return true; + }; + let provider = provider_for(t).provider.to_ascii_lowercase(); + filter.contains(&provider) +} + +pub(crate) fn summary_turns_from_enriched(enriched: &[EnrichedTurn]) -> Vec { + enriched.iter().map(|e| e.turn.clone()).collect() +} + +pub(crate) fn load_summary_by_tool_attribution_turns( + ledger: &crate::ledger::Ledger, + selected: &[EnrichedTurn], + q: &Query, +) -> Result> { + let session_ids: Vec = selected + .iter() + .map(|e| e.turn.session_id.clone()) + .collect::>() + .into_iter() + .collect(); + let turns = ledger.query_turns_in_sessions( + &Query { + source: q.source, + ..Default::default() + }, + &session_ids, + )?; + let mut by_key: IndexMap = IndexMap::new(); + for t in turns { + let key = format!( + "{}|{}|{}", + t.turn.source.wire_str(), + t.turn.session_id, + t.turn.message_id, + ); + by_key.insert(key, t); + } + Ok(by_key.into_values().map(|e| e.turn).collect()) +} + +pub(crate) fn resolve_summary_agent_session_tree( + ledger: &crate::ledger::Ledger, + agent_id: &str, +) -> Result> { + Ok(collect_summary_agent_session_tree( + &ledger.query_relationships(&Query::default())?, + agent_id, + )) +} + +pub(crate) fn collect_summary_agent_session_tree( + relationships: &[SessionRelationshipRecord], + agent_id: &str, +) -> HashSet { + let mut by_parent: HashMap> = HashMap::new(); + for r in relationships { + if r.relationship_type != RelationshipType::Subagent { + continue; + } + let Some(parent) = r.related_session_id.as_deref() else { + continue; + }; + if parent.is_empty() { + continue; + } + by_parent.entry(parent.to_string()).or_default().push(r); + } + + let mut sessions = HashSet::new(); + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([agent_id.to_string()]); + while let Some(parent) = queue.pop_front() { + if !seen.insert(parent.clone()) { + continue; + } + for child in by_parent.get(&parent).into_iter().flatten() { + sessions.insert(child.session_id.clone()); + queue.push_back(child.session_id.clone()); + if let Some(agent) = child.agent_id.as_ref() { + if !agent.is_empty() { + queue.push_back(agent.clone()); + } + } + } + } + sessions +} + +/// Resolve the Claude projects root and run [`count_subagents_under`] +/// against it for the `subagents: X paired, Y orphan` summary line. +/// +/// We honor `BURN_CLAUDE_PROJECTS_DIR` so tests (and integration +/// fixtures) can point at a sandbox without scanning the developer's +/// `~/.claude`. The env var also lets the CLI summary remain +/// reproducible against a fixture-only test suite. When unset we fall +/// back to `$HOME/.claude/projects`; if that doesn't exist the +/// underlying walk returns `(0, 0)` and the summary line is skipped. +/// +/// `session_filter` matches the rest of the summary's filter set: +/// `None` means "no filter — count every session reachable from the +/// projects root" (the un-filtered `burn summary` path); `Some(set)` +/// means "only count sidecars whose session id is in `set`" so a +/// `burn summary --session A` / `--project B` / `--since 24h` run gets +/// a subagent count scoped to the same sessions the rest of the +/// numbers cover. +pub(crate) fn compute_summary_subagent_counts( + session_filter: Option<&HashSet>, +) -> crate::reader::SubagentCounts { + use crate::reader::count_subagents_under; + let root = if let Some(p) = std::env::var_os("BURN_CLAUDE_PROJECTS_DIR") { + std::path::PathBuf::from(p) + } else { + // Defaults to `~/.claude/projects` (HOME, then USERPROFILE on + // Windows — see crate::util::home_dir) when + // `BURN_CLAUDE_PROJECTS_DIR` is unset. + crate::util::home_dir().join(".claude").join("projects") + }; + count_subagents_under(&root, session_filter) +} + +/// Build the session-id filter set the subagent counter should descend +/// into. Returns `None` when `opts` carries no scoping filters, which +/// preserves the original "scan every reachable session" behavior for +/// the bare `burn summary` invocation. Returns `Some(set)` when any +/// filter (`session`, `project`, `since`, `workflow`, `tags`, `agent`, +/// `providers`) is active — `set` is the session ids that survived +/// every filter, derived from the already-filtered `turns` slice. +/// +/// Plumbing the filter via the filtered turn set (instead of e.g. +/// duplicating the SQL filters inside the walker) ensures the count +/// can never diverge from the rest of the summary numbers: anything +/// that drops a session from the row aggregates also drops it from the +/// subagent count. +pub(crate) fn summary_subagent_session_filter( + opts: &SummaryReportOptions, + turns: &[TurnRecord], +) -> Option> { + let has_filter = opts.session.is_some() + || opts.project.is_some() + || opts.since.is_some() + || opts.workflow.is_some() + || opts.agent.is_some() + || opts.tags.as_ref().map(|t| !t.is_empty()).unwrap_or(false) + || opts + .providers + .as_ref() + .map(|p| !p.is_empty()) + .unwrap_or(false); + if !has_filter { + return None; + } + Some(turns.iter().map(|t| t.session_id.clone()).collect()) +} + +pub(crate) fn compute_summary_quality_for_turns( + ledger: &crate::ledger::Ledger, + turns: &[TurnRecord], +) -> Result { + let content_by_session = load_summary_content_for_quality(ledger, turns)?; + Ok(compute_quality( + turns, + &ComputeQualityOptions { + content_by_session: Some(&content_by_session), + now_ms: None, + }, + )) +} + +pub(crate) fn load_summary_content_for_quality( + ledger: &crate::ledger::Ledger, + turns: &[TurnRecord], +) -> Result>> { + let mut seen = HashSet::new(); + let mut out = HashMap::new(); + for t in turns { + if !seen.insert(t.session_id.clone()) { + continue; + } + let records = ledger.query_content(&Query { + session_id: Some(t.session_id.clone()), + ..Default::default() + })?; + if !records.is_empty() { + out.insert(t.session_id.clone(), records); + } + } + Ok(out) +} + +pub(crate) fn summary_aggregate_by_tag( + enriched: &[EnrichedTurn], + tag_key: &str, + pricing: &PricingTable, +) -> (Vec, Vec>) { + let mut by_value: HashMap, UsageCostAggregateRow> = HashMap::new(); + let mut order: Vec> = Vec::new(); + for enriched in enriched { + let value = enriched.enrichment.get(tag_key).cloned(); + let label = value.clone().unwrap_or_else(|| "(untagged)".to_string()); + let row = by_value.entry(value.clone()).or_insert_with(|| { + order.push(value.clone()); + summary_empty_row(&label) + }); + row.turns += 1; + row.usage.input += enriched.turn.usage.input; + row.usage.output += enriched.turn.usage.output; + row.usage.reasoning += enriched.turn.usage.reasoning; + row.usage.cache_read += enriched.turn.usage.cache_read; + row.usage.cache_create_5m += enriched.turn.usage.cache_create_5m; + row.usage.cache_create_1h += enriched.turn.usage.cache_create_1h; + summary_accumulate_coverage( + &mut row.coverage, + enriched.turn.fidelity.as_ref().map(|f| &f.coverage), + ); + if let Some(c) = cost_for_turn(&enriched.turn, pricing) { + row.cost.total += c.total; + row.cost.input += c.input; + row.cost.output += c.output; + row.cost.reasoning += c.reasoning; + row.cost.cache_read += c.cache_read; + row.cost.cache_create += c.cache_create; + } + } + + let mut pairs: Vec<(Option, UsageCostAggregateRow)> = order + .into_iter() + .map(|value| { + let row = by_value.remove(&value).unwrap(); + (value, row) + }) + .collect(); + pairs.sort_by(|a, b| { + b.1.cost + .total + .partial_cmp(&a.1.cost.total) + .unwrap_or(std::cmp::Ordering::Equal) + }); + let (values, rows): (Vec>, Vec) = + pairs.into_iter().unzip(); + (rows, values) +} + +pub(crate) fn summary_aggregate_by_model( + turns: &[TurnRecord], + pricing: &PricingTable, +) -> Vec { + let mut by_model: IndexMap = IndexMap::new(); + for t in turns { + let key = if t.model.is_empty() { + "unknown".to_string() + } else { + t.model.clone() + }; + let row = by_model + .entry(key.clone()) + .or_insert_with(|| summary_empty_row(&key)); + row.turns += 1; + row.usage.input += t.usage.input; + row.usage.output += t.usage.output; + row.usage.reasoning += t.usage.reasoning; + row.usage.cache_read += t.usage.cache_read; + row.usage.cache_create_5m += t.usage.cache_create_5m; + row.usage.cache_create_1h += t.usage.cache_create_1h; + summary_accumulate_coverage(&mut row.coverage, t.fidelity.as_ref().map(|f| &f.coverage)); + if let Some(c) = cost_for_turn(t, pricing) { + row.cost.total += c.total; + row.cost.input += c.input; + row.cost.output += c.output; + row.cost.reasoning += c.reasoning; + row.cost.cache_read += c.cache_read; + row.cost.cache_create += c.cache_create; + } + } + let mut rows: Vec = by_model.into_values().collect(); + rows.sort_by(|a, b| { + b.cost + .total + .partial_cmp(&a.cost.total) + .unwrap_or(std::cmp::Ordering::Equal) + }); + rows +} + +pub(crate) fn summary_provider_to_aggregate_row(p: ProviderAggregateRow) -> UsageCostAggregateRow { + UsageCostAggregateRow { + label: p.label, + turns: p.turns, + usage: p.usage, + cost: p.cost, + coverage: p.coverage, + } +} + +pub(crate) fn summary_empty_row(label: &str) -> UsageCostAggregateRow { + UsageCostAggregateRow { + label: label.to_string(), + turns: 0, + usage: Usage::default(), + cost: CostBreakdown { + model: label.to_string().into(), + total: 0.0, + input: 0.0, + output: 0.0, + reasoning: 0.0, + cache_read: 0.0, + cache_create: 0.0, + }, + coverage: RowCoverage::default(), + } +} + +pub(crate) fn summary_accumulate_coverage(target: &mut RowCoverage, coverage: Option<&Coverage>) { + for f in [ + CoverageField::Input, + CoverageField::Output, + CoverageField::Reasoning, + CoverageField::CacheRead, + CoverageField::CacheCreate, + ] { + let known = match coverage { + None => true, + Some(c) => match f { + CoverageField::Input => c.has_input_tokens, + CoverageField::Output => c.has_output_tokens, + CoverageField::Reasoning => c.has_reasoning_tokens, + CoverageField::CacheRead => c.has_cache_read_tokens, + CoverageField::CacheCreate => c.has_cache_create_tokens, + }, + }; + let slot = target.field_mut(f); + if known { + slot.known += 1; + } else { + slot.missing += 1; + } + } +} + +pub(crate) fn summary_cell_is_partial(c: &FieldCoverage) -> bool { + c.known > 0 && c.missing > 0 +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct SummaryToolAgg { + pub(crate) calls: u64, + pub(crate) cost: f64, + pub(crate) sized_cost: f64, + pub(crate) even_split_cost: f64, +} + +#[derive(Debug, Default)] +pub(crate) struct SummaryUserTurnSizeBucket { + tool_bytes_by_id: HashMap, + total_bytes: u64, +} + +pub(crate) fn compute_summary_by_tool_report( + ledger: &crate::ledger::Ledger, + turns: &[TurnRecord], + attribution_turns: &[TurnRecord], + pricing: &PricingTable, +) -> Result { + let user_turns_by_session = load_summary_user_turns_for_by_tool(ledger, attribution_turns)?; + let selected_turns = selected_summary_turn_keys(turns); + let (by_tool, unattributed_cost) = attribute_summary_cost_to_tools( + attribution_turns, + pricing, + &user_turns_by_session, + Some(&selected_turns), + ); + let fidelity = summarize_fidelity(turns); + let replacement_savings = summarize_replacement_savings(turns, None); + let mut sorted: Vec<(String, SummaryToolAgg)> = by_tool.into_iter().collect(); + sorted.sort_by(|a, b| { + b.1.cost + .partial_cmp(&a.1.cost) + .unwrap_or(std::cmp::Ordering::Equal) + }); + let rows = sorted + .into_iter() + .map(|(tool, agg)| SummaryToolAttributionRow { + savings: replacement_savings.by_tool.get(&tool).cloned(), + tool, + calls: agg.calls, + attributed_cost: agg.cost, + attribution_method: summary_tool_attribution_method(&agg), + }) + .collect(); + Ok(SummaryByToolReport { + turn_count: turns.len() as u64, + rows, + unattributed_cost, + fidelity, + replacement_savings, + }) +} + +pub(crate) fn load_summary_user_turns_for_by_tool( + ledger: &crate::ledger::Ledger, + turns: &[TurnRecord], +) -> Result>> { + let session_ids: BTreeSet = turns.iter().map(|t| t.session_id.clone()).collect(); + let mut out = HashMap::new(); + for session_id in session_ids { + let rows = ledger.query_user_turns(&Query { + session_id: Some(session_id.clone()), + ..Default::default() + })?; + if !rows.is_empty() { + out.insert(session_id, rows); + } + } + Ok(out) +} + +pub(crate) fn selected_summary_turn_keys(turns: &[TurnRecord]) -> HashSet { + turns.iter().map(summary_turn_identity_key).collect() +} + +pub(crate) fn attribute_summary_cost_to_tools( + turns: &[TurnRecord], + pricing: &PricingTable, + user_turns_by_session: &HashMap>, + selected_turns: Option<&HashSet>, +) -> (IndexMap, f64) { + let mut by_tool: IndexMap = IndexMap::new(); + let mut unattributed = 0.0; + let mut by_session: IndexMap> = IndexMap::new(); + for t in turns { + by_session.entry(t.session_id.clone()).or_default().push(t); + } + + for (session_id, mut list) in by_session { + list.sort_by_key(|t| t.turn_index); + let user_turn_size_index = index_summary_user_turn_block_sizes( + user_turns_by_session + .get(&session_id) + .map(Vec::as_slice) + .unwrap_or(&[]), + ); + for i in 0..list.len() { + let turn = list[i]; + if !summary_turn_is_selected(turn, selected_turns) { + continue; + } + let Some(c) = cost_for_turn(turn, pricing) else { + continue; + }; + let ingest_cost = c.input + c.cache_read + c.cache_create; + + if i == 0 { + unattributed += ingest_cost; + continue; + } + let prior = list[i - 1]; + if prior.tool_calls.is_empty() { + unattributed += ingest_cost; + continue; + } + + let key = summary_bridge_key(&prior.message_id, &turn.message_id); + let sizes = user_turn_size_index.get(&key); + let sized_bytes: u64 = match sizes { + Some(s) => prior + .tool_calls + .iter() + .map(|tc| *s.tool_bytes_by_id.get(&tc.id).unwrap_or(&0)) + .sum(), + None => 0, + }; + if let Some(sizes) = sizes.filter(|_| sized_bytes > 0) { + let allocatable_cost = if sizes.total_bytes > 0 { + ingest_cost * (sized_bytes as f64 / sizes.total_bytes as f64).min(1.0) + } else { + ingest_cost + }; + unattributed += ingest_cost - allocatable_cost; + let mut raw_shares: Vec<(String, f64)> = Vec::new(); + for tc in &prior.tool_calls { + let bytes = *sizes.tool_bytes_by_id.get(&tc.id).unwrap_or(&0); + if bytes == 0 { + continue; + } + by_tool.entry(tc.name.clone()).or_default().calls += 1; + raw_shares.push(( + tc.name.clone(), + (bytes as f64 / sized_bytes as f64) * allocatable_cost, + )); + } + let raw_subtotal: f64 = raw_shares.iter().map(|(_, cost)| *cost).sum(); + let scale = if raw_subtotal > allocatable_cost && raw_subtotal > 0.0 { + allocatable_cost / raw_subtotal + } else { + 1.0 + }; + for (tool, cost) in raw_shares { + let share = cost * scale; + let agg = by_tool.entry(tool).or_default(); + agg.cost += share; + agg.sized_cost += share; + } + } else { + let share = ingest_cost / prior.tool_calls.len() as f64; + for tc in &prior.tool_calls { + let agg = by_tool.entry(tc.name.clone()).or_default(); + agg.calls += 1; + agg.cost += share; + agg.even_split_cost += share; + } + } + } + } + + (by_tool, unattributed) +} + +pub(crate) fn summary_turn_is_selected( + turn: &TurnRecord, + selected_turns: Option<&HashSet>, +) -> bool { + selected_turns + .map(|keys| keys.contains(&summary_turn_identity_key(turn))) + .unwrap_or(true) +} + +pub(crate) fn summary_turn_identity_key(turn: &TurnRecord) -> String { + format!( + "{}\0{}\0{}", + turn.source.wire_str(), + turn.session_id, + turn.message_id + ) +} + +pub(crate) fn index_summary_user_turn_block_sizes( + user_turns: &[UserTurnRecord], +) -> HashMap { + let mut out: HashMap = HashMap::new(); + for user_turn in user_turns { + let (Some(preceding), Some(following)) = ( + user_turn.preceding_message_id.as_ref(), + user_turn.following_message_id.as_ref(), + ) else { + continue; + }; + let bucket = out + .entry(summary_bridge_key(preceding, following)) + .or_default(); + for block in &user_turn.blocks { + let bytes = block.byte_len; + bucket.total_bytes += bytes; + if block.kind != UserTurnBlockKind::ToolResult { + continue; + } + let Some(tool_use_id) = block.tool_use_id.as_ref() else { + continue; + }; + *bucket + .tool_bytes_by_id + .entry(tool_use_id.clone()) + .or_default() += bytes; + } + } + out +} + +pub(crate) fn summary_bridge_key(preceding_message_id: &str, following_message_id: &str) -> String { + format!("{preceding_message_id}\0{following_message_id}") +} + +pub(crate) fn summary_tool_attribution_method( + agg: &SummaryToolAgg, +) -> SummaryToolAttributionMethod { + if agg.sized_cost == 0.0 && agg.even_split_cost == 0.0 { + SummaryToolAttributionMethod::Unattributed + } else if agg.sized_cost >= agg.even_split_cost { + SummaryToolAttributionMethod::Sized + } else { + SummaryToolAttributionMethod::EvenSplit + } +} + +pub(crate) const SUMMARY_RELATIONSHIP_ORDER: [RelationshipType; 4] = [ + RelationshipType::Root, + RelationshipType::Continuation, + RelationshipType::Fork, + RelationshipType::Subagent, +]; + +#[derive(Debug, Clone)] +pub(crate) struct SummaryRelationshipMatch { + pub(crate) relationship_type: RelationshipType, + pub(crate) session_id: String, + pub(crate) subagent_type: Option, + pub(crate) turn_count: u64, + pub(crate) cost: f64, +} + +pub(crate) struct SummaryRelationshipTurnIndex<'a> { + all_by_session: HashMap>, + main_by_session: HashMap>, + sidechain_by_session: HashMap>, + subagent_by_session_agent: HashMap>, +} + +pub(crate) fn summary_relationship_query_for_turn_slice(q: &Query) -> Query { + Query { + session_id: q.session_id.clone(), + source: q.source, + ..Default::default() + } +} + +pub(crate) fn match_summary_relationships_to_turns( + relationships: &[SessionRelationshipRecord], + turns: &[TurnRecord], + pricing: &PricingTable, +) -> Vec { + let index = build_summary_relationship_turn_index(turns); + let mut out = Vec::new(); + let mut seen = HashSet::new(); + for r in relationships { + let key = summary_relationship_instance_key(r); + if !seen.insert(key) { + continue; + } + let matched_turns = summary_turns_for_relationship(r, &index); + if matched_turns.is_empty() { + continue; + } + let cost = matched_turns + .iter() + .map(|t| cost_for_turn(t, pricing).map(|c| c.total).unwrap_or(0.0)) + .sum(); + out.push(SummaryRelationshipMatch { + relationship_type: r.relationship_type, + session_id: r.session_id.clone(), + subagent_type: summary_relationship_subagent_type(r, &matched_turns), + turn_count: matched_turns.len() as u64, + cost, + }); + } + out +} + +pub(crate) fn build_summary_relationship_turn_index( + turns: &[TurnRecord], +) -> SummaryRelationshipTurnIndex<'_> { + let mut index = SummaryRelationshipTurnIndex { + all_by_session: HashMap::new(), + main_by_session: HashMap::new(), + sidechain_by_session: HashMap::new(), + subagent_by_session_agent: HashMap::new(), + }; + for turn in turns { + index + .all_by_session + .entry(turn.session_id.clone()) + .or_default() + .push(turn); + if summary_is_main_thread_turn(turn) { + index + .main_by_session + .entry(turn.session_id.clone()) + .or_default() + .push(turn); + } + if turn + .subagent + .as_ref() + .map(|s| s.is_sidechain) + .unwrap_or(false) + { + index + .sidechain_by_session + .entry(turn.session_id.clone()) + .or_default() + .push(turn); + } + if let Some(agent_id) = turn.subagent.as_ref().and_then(|s| s.agent_id.as_ref()) { + if !agent_id.is_empty() { + index + .subagent_by_session_agent + .entry(summary_session_agent_key(&turn.session_id, agent_id)) + .or_default() + .push(turn); + } + } + } + index +} + +pub(crate) fn summary_turns_for_relationship<'a>( + r: &SessionRelationshipRecord, + index: &'a SummaryRelationshipTurnIndex<'a>, +) -> Vec<&'a TurnRecord> { + match r.relationship_type { + RelationshipType::Root => index + .main_by_session + .get(&r.session_id) + .cloned() + .unwrap_or_default(), + RelationshipType::Subagent => { + if let Some(agent_id) = r.agent_id.as_ref().filter(|s| !s.is_empty()) { + let key = summary_session_agent_key(&r.session_id, agent_id); + if let Some(direct) = index.subagent_by_session_agent.get(&key) { + if !direct.is_empty() { + return direct.clone(); + } + } + if r.session_id == *agent_id { + return index + .all_by_session + .get(&r.session_id) + .cloned() + .unwrap_or_default(); + } + } + if let Some(sidechain) = index.sidechain_by_session.get(&r.session_id) { + if !sidechain.is_empty() { + return sidechain.clone(); + } + } + if r.source.wire_str() == "spawn-env" { + return index + .all_by_session + .get(&r.session_id) + .cloned() + .unwrap_or_default(); + } + Vec::new() + } + RelationshipType::Continuation | RelationshipType::Fork => index + .all_by_session + .get(&r.session_id) + .cloned() + .unwrap_or_default(), + } +} + +pub(crate) fn aggregate_summary_relationship_stats( + matches: &[SummaryRelationshipMatch], +) -> Vec { + #[derive(Default)] + struct RelationshipSessionRollup { + relationship_count: u64, + turn_count: u64, + cost: f64, + } + + let mut by_type: HashMap> = + HashMap::new(); + for m in matches { + let by_session = by_type.entry(m.relationship_type).or_default(); + let current = by_session.entry(m.session_id.clone()).or_default(); + current.relationship_count += 1; + current.turn_count += m.turn_count; + current.cost += m.cost; + } + + let mut out = Vec::new(); + for relationship_type in SUMMARY_RELATIONSHIP_ORDER { + let Some(by_session) = by_type.get(&relationship_type) else { + continue; + }; + if by_session.is_empty() { + continue; + } + let mut costs: Vec = by_session.values().map(|rollup| rollup.cost).collect(); + costs.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let total_cost: f64 = costs.iter().sum(); + let session_count = by_session.len() as u64; + out.push(SummaryRelationshipStats { + relationship_type, + count: by_session + .values() + .map(|rollup| rollup.relationship_count) + .sum(), + session_count, + turn_count: by_session.values().map(|rollup| rollup.turn_count).sum(), + total_cost, + median_cost: summary_percentile(&costs, 0.5), + p95_cost: summary_percentile(&costs, 0.95), + mean_cost: if session_count > 0 { + total_cost / session_count as f64 + } else { + 0.0 + }, + }); + } + out +} + +pub(crate) fn aggregate_summary_relationship_subagent_stats( + matches: &[SummaryRelationshipMatch], +) -> Vec { + struct Agg { + turns: u64, + total: f64, + costs: Vec, + } + let mut by_type: IndexMap = IndexMap::new(); + for m in matches { + if m.relationship_type != RelationshipType::Subagent { + continue; + } + let ty = m + .subagent_type + .clone() + .unwrap_or_else(|| "(unknown)".to_string()); + let agg = by_type.entry(ty).or_insert_with(|| Agg { + turns: 0, + total: 0.0, + costs: Vec::new(), + }); + agg.turns += m.turn_count; + agg.total += m.cost; + agg.costs.push(m.cost); + } + + let mut out = Vec::new(); + for (subagent_type, mut agg) in by_type { + agg.costs + .sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let invocations = agg.costs.len() as u64; + out.push(SummaryRelationshipSubagentStats { + subagent_type, + invocations, + turns: agg.turns, + total_cost: agg.total, + median_cost: summary_percentile(&agg.costs, 0.5), + p95_cost: summary_percentile(&agg.costs, 0.95), + mean_cost: if invocations > 0 { + agg.total / invocations as f64 + } else { + 0.0 + }, + }); + } + out.sort_by(|a, b| { + b.total_cost + .partial_cmp(&a.total_cost) + .unwrap_or(std::cmp::Ordering::Equal) + }); + out +} + +pub(crate) fn summary_relationship_subagent_type( + relationship: &SessionRelationshipRecord, + turns: &[&TurnRecord], +) -> Option { + if let Some(st) = &relationship.subagent_type { + return Some(st.clone()); + } + turns.iter().find_map(|t| { + t.subagent + .as_ref() + .and_then(|s| s.subagent_type.as_ref()) + .cloned() + }) +} + +pub(crate) fn summary_relationship_instance_key(r: &SessionRelationshipRecord) -> String { + [ + r.source.wire_str(), + r.relationship_type.wire_str(), + &r.session_id, + r.related_session_id.as_deref().unwrap_or(""), + r.agent_id.as_deref().unwrap_or(""), + r.parent_tool_use_id.as_deref().unwrap_or(""), + ] + .join("\0") +} + +pub(crate) fn summary_session_agent_key(session_id: &str, agent_id: &str) -> String { + format!("{session_id}\0{agent_id}") +} + +pub(crate) fn summary_is_main_thread_turn(turn: &TurnRecord) -> bool { + match &turn.subagent { + None => true, + Some(sub) => !sub.is_sidechain || sub.agent_id.as_deref() == Some(&turn.session_id), + } +} + +pub(crate) fn summary_percentile(sorted: &[f64], p: f64) -> f64 { + if sorted.is_empty() { + return 0.0; + } + let rank = + ((p * sorted.len() as f64).ceil() as i64 - 1).clamp(0, sorted.len() as i64 - 1) as usize; + sorted[rank] +} + +pub(crate) fn collect_summary_subagent_tree_relationships( + ledger: &crate::ledger::Ledger, + session_id: &str, + q: &Query, +) -> Result> { + let relationships = ledger.query_relationships(&Query { + source: q.source, + ..Default::default() + })?; + Ok(collect_summary_connected_relationships( + &relationships, + session_id, + )) +} + +pub(crate) fn collect_summary_connected_relationships( + relationships: &[SessionRelationshipRecord], + session_id: &str, +) -> Vec { + let mut by_id: HashMap> = HashMap::new(); + for (idx, r) in relationships.iter().enumerate() { + for id in summary_relationship_connected_ids(r) { + if !id.is_empty() { + by_id.entry(id).or_default().push(idx); + } + } + } + + let mut out: IndexMap = IndexMap::new(); + let mut seen_ids = HashSet::new(); + let mut queue = VecDeque::from([session_id.to_string()]); + while let Some(id) = queue.pop_front() { + if !seen_ids.insert(id.clone()) { + continue; + } + let Some(rows) = by_id.get(&id) else { + continue; + }; + for idx in rows { + let r = &relationships[*idx]; + for next in summary_relationship_connected_ids(r) { + if !next.is_empty() && !seen_ids.contains(&next) { + queue.push_back(next); + } + } + out.insert(summary_relationship_instance_key(r), r.clone()); + } + } + out.into_values().collect() +} + +pub(crate) fn summary_relationship_connected_ids(r: &SessionRelationshipRecord) -> Vec { + let mut ids = vec![r.session_id.clone()]; + if let Some(related) = &r.related_session_id { + ids.push(related.clone()); + } + if let Some(agent) = &r.agent_id { + ids.push(agent.clone()); + } + ids +} + +pub(crate) fn load_summary_subagent_tree_turns( + ledger: &crate::ledger::Ledger, + session_id: &str, + relationships: &[SessionRelationshipRecord], + q: &Query, +) -> Result> { + let mut session_ids = HashSet::from([session_id.to_string()]); + for r in relationships { + session_ids.insert(r.session_id.clone()); + } + + let mut by_key: IndexMap = IndexMap::new(); + for id in session_ids { + let turns = ledger.query_turns(&Query { + session_id: Some(id), + ..q.clone() + })?; + for t in turns { + let key = format!( + "{}|{}|{}", + t.turn.source.wire_str(), + t.turn.session_id, + t.turn.message_id, + ); + by_key.insert(key, t); + } + } + Ok(by_key.into_values().collect()) +} + +pub(crate) fn find_summary_tree_node<'a>( + trees: impl IntoIterator, + node_id: &str, +) -> Option { + for root in trees { + if let Some(found) = find_summary_node(root, node_id) { + return Some(found.clone()); + } + } + None +} + +pub(crate) fn find_summary_node<'a>( + node: &'a SubagentTreeNode, + node_id: &str, +) -> Option<&'a SubagentTreeNode> { + if node.node_id == node_id { + return Some(node); + } + for child in &node.children { + if let Some(found) = find_summary_node(child, node_id) { + return Some(found); + } + } + None +} diff --git a/crates/relayburn-sdk/src/query_verbs/summary/mod.rs b/crates/relayburn-sdk/src/query_verbs/summary/mod.rs new file mode 100644 index 00000000..cf692e26 --- /dev/null +++ b/crates/relayburn-sdk/src/query_verbs/summary/mod.rs @@ -0,0 +1,970 @@ +use super::*; + +// --------------------------------------------------------------------------- +// summary +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryOptions { + pub session: Option, + pub project: Option, + pub since: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tags: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub group_by_tag: Option, + pub ledger_home: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryToolRow { + pub tool: String, + pub tokens: u64, + pub cost: f64, + pub count: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryModelRow { + pub model: String, + pub tokens: u64, + pub cost: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryTagRow { + pub tag: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option, + pub tokens: u64, + pub cost: f64, + pub turn_count: u64, +} + +/// Per-outcome turn counts, surfaced by `burn summary` for the one-line +/// outcome breakdown (`142 end_turn, 3 max_tokens, 1 refusal, 0 pause`). +/// +/// Counts mirror the [`StopReason`] enum variants plus a `none` slot for +/// turns whose row carried no `stop_reason` field at all — that's Codex +/// today (no field in the rollout schema) and any pre-3.0 ledger row that +/// was ingested before the reader started populating the enum. +/// +/// `Silent` is reserved for "row exists, carries a stop_reason that we +/// don't recognize" — distinct from `none` so we can spot a future harness +/// regression rather than silently lumping it with Codex. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StopReasonCounts { + pub end_turn: u64, + pub max_tokens: u64, + pub pause_turn: u64, + pub stop_sequence: u64, + pub tool_use: u64, + pub refusal: u64, + pub silent: u64, + /// Turns whose record carried no `stop_reason` field — e.g. Codex + /// rollouts (the harness doesn't report one) or pre-3.0 ledger rows + /// from before the reader started parsing the field. + pub none: u64, +} + +impl StopReasonCounts { + /// Accumulate one turn's outcome into the bucket counts. `None` lands + /// in [`Self::none`]; unrecognized variants would already be normalized + /// to [`StopReason::Silent`] upstream by the lenient deserializer. + pub fn bump(&mut self, reason: Option) { + match reason { + None => self.none += 1, + Some(StopReason::EndTurn) => self.end_turn += 1, + Some(StopReason::MaxTokens) => self.max_tokens += 1, + Some(StopReason::PauseTurn) => self.pause_turn += 1, + Some(StopReason::StopSequence) => self.stop_sequence += 1, + Some(StopReason::ToolUse) => self.tool_use += 1, + Some(StopReason::Refusal) => self.refusal += 1, + Some(StopReason::Silent) => self.silent += 1, + } + } + + /// Fold every turn's `stop_reason` into a fresh counts struct. + pub fn from_turns(turns: &[TurnRecord]) -> Self { + let mut out = Self::default(); + for t in turns { + out.bump(t.stop_reason); + } + out + } + + /// True iff every counter is zero — useful for "skip the outcome line + /// entirely" presentation logic in summary. + pub fn is_empty(&self) -> bool { + self.end_turn + | self.max_tokens + | self.pause_turn + | self.stop_sequence + | self.tool_use + | self.refusal + | self.silent + | self.none + == 0 + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Summary { + pub total_tokens: u64, + pub total_cost: f64, + pub turn_count: u64, + pub by_tool: Vec, + pub by_model: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub by_tag: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replacement_savings: Option, + /// Per-outcome breakdown — `end_turn` / `max_tokens` / `refusal` / etc. + /// Counts roll up the trailing `stop_reason` of every assistant turn + /// in the filtered slice. See #437. + pub stop_reasons: StopReasonCounts, + /// Count of turns whose model had no entry in the pricing snapshot. + /// Their cost is reported as $0. Zero when all models are priced. + #[serde(default)] + pub unpriced_turns: u64, + /// Distinct model names (first-seen order) that had no pricing entry. + /// Empty when all models are priced. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub unpriced_models: Vec, +} + +impl LedgerHandle { + pub fn summary(&self, opts: SummaryOptions) -> Result { + let mut q = build_query( + opts.session.as_deref(), + opts.project.as_deref(), + opts.since.as_deref(), + )?; + if let Some(tags) = opts.tags.clone() { + validate_tags(&tags)?; + if !tags.is_empty() { + q.enrichment = Some(tags); + } + } + let group_by_tag = opts.group_by_tag.clone(); + if let Some(tag) = group_by_tag.as_deref() { + validate_tag_key(tag, "groupByTag")?; + } + let enriched = self.inner.query_turns(&q)?; + let turns: Vec = enriched.iter().map(|e| e.turn.clone()).collect(); + let pricing = load_pricing(None); + let mut summary = compute_summary(&turns, &pricing); + if let Some(tag) = group_by_tag { + summary.by_tag = Some(compute_summary_by_tag(&enriched, &tag, &pricing)); + } + Ok(summary) + } +} + +pub fn summary(opts: SummaryOptions) -> Result { + let handle = open_with(opts.ledger_home.as_deref())?; + handle.summary(SummaryOptions { + ledger_home: None, + ..opts + }) +} + +pub(crate) fn validate_tags(tags: &Enrichment) -> Result<()> { + for key in tags.keys() { + validate_tag_key(key, "tag")?; + } + Ok(()) +} + +pub(crate) fn validate_tag_key(key: &str, label: &str) -> Result<()> { + if key.is_empty() { + anyhow::bail!("{label} key must be non-empty"); + } + Ok(()) +} + +pub(crate) fn compute_summary(turns: &[TurnRecord], pricing: &PricingTable) -> Summary { + // First-seen iteration order matches TS `Map` semantics. + let mut by_tool_order: Vec = Vec::new(); + let mut by_tool: HashMap = HashMap::new(); + let mut by_model_order: Vec = Vec::new(); + let mut by_model: HashMap = HashMap::new(); + let mut total_tokens: u64 = 0; + let mut total_cost: f64 = 0.0; + + for t in turns { + let cost = cost_for_turn(t, pricing).map(|c| c.total).unwrap_or(0.0); + let tokens = t.usage.input + + t.usage.output + + t.usage.reasoning + + t.usage.cache_read + + t.usage.cache_create_5m + + t.usage.cache_create_1h; + total_tokens += tokens; + total_cost += cost; + + let model_row = by_model.entry(t.model.clone()).or_insert_with(|| { + by_model_order.push(t.model.clone()); + SummaryModelRow { + model: t.model.clone(), + tokens: 0, + cost: 0.0, + } + }); + model_row.tokens += tokens; + model_row.cost += cost; + + for call in &t.tool_calls { + let tool_row = by_tool.entry(call.name.clone()).or_insert_with(|| { + by_tool_order.push(call.name.clone()); + SummaryToolRow { + tool: call.name.clone(), + tokens: 0, + cost: 0.0, + count: 0, + } + }); + tool_row.tokens += tokens; + tool_row.cost += cost; + tool_row.count += 1; + } + } + + let savings = summarize_replacement_savings(turns, None); + let replacement_savings = if savings.calls > 0 { + Some(savings) + } else { + None + }; + + // Use the same pricing table that was used for cost accumulation so the + // count precisely matches which turns contributed $0 to `total_cost`. + let (unpriced_turns, unpriced_models) = tally_unpriced(turns, pricing); + + Summary { + total_tokens, + total_cost, + turn_count: turns.len() as u64, + by_tool: by_tool_order + .into_iter() + .map(|k| by_tool.remove(&k).unwrap()) + .collect(), + by_model: by_model_order + .into_iter() + .map(|k| by_model.remove(&k).unwrap()) + .collect(), + by_tag: None, + replacement_savings, + stop_reasons: StopReasonCounts::from_turns(turns), + unpriced_turns, + unpriced_models, + } +} + +fn compute_summary_by_tag( + enriched: &[EnrichedTurn], + tag: &str, + pricing: &PricingTable, +) -> Vec { + let mut order: Vec> = Vec::new(); + let mut rows: HashMap, SummaryTagRow> = HashMap::new(); + + for e in enriched { + let value = e.enrichment.get(tag).cloned(); + let tokens = total_tokens_for_turn(&e.turn); + let cost = cost_for_turn(&e.turn, pricing) + .map(|c| c.total) + .unwrap_or(0.0); + let row = rows.entry(value.clone()).or_insert_with(|| { + order.push(value.clone()); + SummaryTagRow { + tag: tag.to_string(), + value, + tokens: 0, + cost: 0.0, + turn_count: 0, + } + }); + row.tokens += tokens; + row.cost += cost; + row.turn_count += 1; + } + + let mut out: Vec = order + .into_iter() + .map(|k| rows.remove(&k).unwrap()) + .collect(); + out.sort_by(|a, b| { + b.cost + .partial_cmp(&a.cost) + .unwrap_or(std::cmp::Ordering::Equal) + }); + out +} + +fn total_tokens_for_turn(t: &TurnRecord) -> u64 { + t.usage.input + + t.usage.output + + t.usage.reasoning + + t.usage.cache_read + + t.usage.cache_create_5m + + t.usage.cache_create_1h +} + +// --------------------------------------------------------------------------- +// richer summary report +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryReportOptions { + pub session: Option, + pub project: Option, + pub since: Option, + pub workflow: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tags: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub group_by_tag: Option, + pub agent: Option, + /// Provider labels to keep. Values are trimmed and matched + /// case-insensitively against the SDK's effective provider resolver. + #[serde(default)] + pub providers: Option>, + #[serde(default)] + pub mode: SummaryReportMode, + #[serde(default)] + pub include_quality: bool, + pub ledger_home: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", tag = "kind")] +pub enum SummaryReportMode { + Grouped { + #[serde(default)] + by_provider: bool, + }, + ByTool, + BySubagentType, + ByRelationship { + #[serde(default)] + subagent: bool, + }, + SubagentTree { + #[serde(default)] + session_id: Option, + }, +} + +impl Default for SummaryReportMode { + fn default() -> Self { + Self::Grouped { by_provider: false } + } +} + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum SummaryGroupBy { + Model, + Provider, + Tag, +} + +impl SummaryGroupBy { + pub fn wire_str(self) -> &'static str { + match self { + Self::Model => "model", + Self::Provider => "provider", + Self::Tag => "tag", + } + } + + pub fn json_key(self) -> &'static str { + match self { + Self::Model => "byModel", + Self::Provider => "byProvider", + Self::Tag => "byTag", + } + } +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +#[allow(clippy::large_enum_variant)] +pub enum SummaryReport { + Grouped(SummaryGroupedReport), + ByTool(SummaryByToolReport), + BySubagentType(SummarySubagentTypeReport), + Relationship(SummaryRelationshipReport), + SubagentTree(SummarySubagentTreeReport), +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryGroupedReport { + pub group_by: SummaryGroupBy, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tag_key: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub tag_values: Vec>, + pub turn_count: u64, + pub rows: Vec, + pub total_cost: CostBreakdown, + pub fidelity: FidelitySummary, + /// Stable TS-compatible JSON shape for per-cell coverage. Kept in the SDK + /// so presenters don't rebuild order-sensitive HashMap projections. + pub per_cell_fidelity: serde_json::Value, + pub replacement_savings: ReplacementSavingsSummary, + /// Per-outcome turn counts (issue #437). Always populated; presenters + /// decide whether to render the line based on `is_empty()`. + pub stop_reasons: StopReasonCounts, + /// Paired / orphan subagent transcript counts (issue #435). Populated + /// by a lazy walk over the Claude `~/.claude/projects/` tree at + /// summary time — when no sidecars exist anywhere reachable the + /// `read_dir` short-circuits and the field stays at + /// `SubagentCounts::default()`. Presenters render the + /// `subagents: X paired, Y orphan` line only when + /// `!subagents.is_empty()`. + #[serde( + default, + skip_serializing_if = "crate::reader::SubagentCounts::is_empty" + )] + pub subagents: crate::reader::SubagentCounts, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub quality: Option, + /// Count of turns whose model had no entry in the pricing snapshot. + /// Their cost is reported as $0. Zero when all models are priced. + #[serde(default)] + pub unpriced_turns: u64, + /// Distinct model names (first-seen order) that had no pricing entry. + /// Empty when all models are priced. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub unpriced_models: Vec, +} + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum SummaryToolAttributionMethod { + Unattributed, + Sized, + EvenSplit, +} + +impl SummaryToolAttributionMethod { + pub fn wire_str(self) -> &'static str { + match self { + Self::Unattributed => "unattributed", + Self::Sized => "sized", + Self::EvenSplit => "even-split", + } + } +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryToolAttributionRow { + pub tool: String, + pub calls: u64, + pub attributed_cost: f64, + pub attribution_method: SummaryToolAttributionMethod, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub savings: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryByToolReport { + pub turn_count: u64, + pub rows: Vec, + pub unattributed_cost: f64, + pub fidelity: FidelitySummary, + pub replacement_savings: ReplacementSavingsSummary, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SummarySubagentTypeReport { + pub stats: Vec, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryRelationshipReport { + pub relationships: Vec, + pub subagent_types: Vec, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryRelationshipStats { + pub relationship_type: RelationshipType, + pub count: u64, + pub session_count: u64, + pub turn_count: u64, + pub total_cost: f64, + pub median_cost: f64, + pub p95_cost: f64, + pub mean_cost: f64, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryRelationshipSubagentStats { + pub subagent_type: String, + pub invocations: u64, + pub turns: u64, + pub total_cost: f64, + pub median_cost: f64, + pub p95_cost: f64, + pub mean_cost: f64, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SummarySubagentTreeReport { + pub session_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub root: Option, +} + +/// One time bucket of a [`SummaryTimeseries`]: the grouped summary totals for +/// turns whose `ts` falls in `[start, end)`. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryBucket { + pub start: String, + pub end: String, + pub turn_count: u64, + pub total_tokens: u64, + pub total_cost: CostBreakdown, + pub group_by: SummaryGroupBy, + pub rows: Vec, +} + +/// A time-series of grouped summary totals — one [`SummaryBucket`] per +/// `bucket_secs`-wide window across the `--since` range. Produced by +/// [`LedgerHandle::summary_timeseries`]. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SummaryTimeseries { + #[serde(rename = "bucketSeconds")] + pub bucket_secs: u64, + pub buckets: Vec, +} + +impl LedgerHandle { + /// Time-bucketed cost/usage totals (the `--bucket` form of the default + /// grouped summary). Fetches the `--since` window once, then partitions the + /// turns by `ts` into `bucket_secs`-wide buckets and aggregates each — a + /// pure per-turn fold, so per-bucket totals sum back to the un-bucketed + /// total. Supported only for the default grouped (`byModel`/`byProvider`) + /// summary; the tool/subagent/relationship attribution modes are rejected. + pub fn summary_timeseries( + &self, + opts: SummaryReportOptions, + bucket_secs: u64, + ) -> Result { + let by_provider = match &opts.mode { + SummaryReportMode::Grouped { by_provider } => *by_provider, + _ => anyhow::bail!( + "--bucket is only supported with the default grouped summary, not \ + --by-tool/--by-subagent-type/--by-relationship/--subagent-tree" + ), + }; + if opts.group_by_tag.is_some() { + anyhow::bail!("--bucket is not supported with --group-by-tag"); + } + if opts.include_quality { + anyhow::bail!("--bucket is not supported with --quality metrics yet"); + } + + let q = build_summary_report_query(&opts)?; + let provider_filter = normalize_summary_provider_filter(opts.providers.as_deref()); + let pricing = load_pricing(None); + let agent_session_ids = match opts.agent.as_deref() { + Some(agent_id) => Some(resolve_summary_agent_session_tree(&self.inner, agent_id)?), + None => None, + }; + + let enriched = self.inner.query_turns(&q)?; + let enriched = filter_summary_enriched_turns( + enriched, + opts.agent.as_deref(), + agent_session_ids.as_ref(), + provider_filter.as_ref(), + ); + let turns = summary_turns_from_enriched(&enriched); + + let Some((buckets, per_bucket)) = + super::partition_into_buckets(turns, q.since.as_deref(), bucket_secs, |t| &t.ts)? + else { + return Ok(SummaryTimeseries { + bucket_secs, + buckets: Vec::new(), + }); + }; + + let group_by = if by_provider { + SummaryGroupBy::Provider + } else { + SummaryGroupBy::Model + }; + let out = per_bucket + .into_iter() + .enumerate() + .map(|(i, bturns)| { + let rows = if by_provider { + aggregate_by_provider(&bturns, AggregateByProviderOptions::new(&pricing)) + .into_iter() + .map(summary_provider_to_aggregate_row) + .collect::>() + } else { + summary_aggregate_by_model(&bturns, &pricing) + }; + let total_cost = sum_costs(rows.iter().map(|r| &r.cost)); + let total_tokens: u64 = bturns.iter().map(total_tokens_for_turn).sum(); + SummaryBucket { + start: buckets.start_iso(i), + end: buckets.end_iso(i), + turn_count: bturns.len() as u64, + total_tokens, + total_cost, + group_by, + rows, + } + }) + .collect(); + + Ok(SummaryTimeseries { + bucket_secs, + buckets: out, + }) + } + + pub fn summary_report(&self, opts: SummaryReportOptions) -> Result { + let q = build_summary_report_query(&opts)?; + let provider_filter = normalize_summary_provider_filter(opts.providers.as_deref()); + let pricing = load_pricing(None); + let agent_session_ids = match opts.agent.as_deref() { + Some(agent_id) => Some(resolve_summary_agent_session_tree(&self.inner, agent_id)?), + None => None, + }; + + if let SummaryReportMode::SubagentTree { session_id } = &opts.mode { + let session_id = session_id + .as_deref() + .filter(|s| !s.is_empty()) + .map(str::to_string) + .or_else(|| q.session_id.clone()) + .ok_or_else(|| anyhow::anyhow!("subagent tree summary requires a session id"))?; + let relationships = + collect_summary_subagent_tree_relationships(&self.inner, &session_id, &q)?; + let enriched = + load_summary_subagent_tree_turns(&self.inner, &session_id, &relationships, &q)?; + let enriched = filter_summary_enriched_turns( + enriched, + opts.agent.as_deref(), + agent_session_ids.as_ref(), + provider_filter.as_ref(), + ); + let turns = summary_turns_from_enriched(&enriched); + let tree_opts = + BuildSubagentTreeOptions::new(&pricing).with_relationships(&relationships); + let trees = build_subagent_tree(&turns, &tree_opts); + let root = trees + .get(&session_id) + .cloned() + .or_else(|| find_summary_tree_node(trees.values(), &session_id)); + return Ok(SummaryReport::SubagentTree(SummarySubagentTreeReport { + session_id, + root, + })); + } + + let enriched = self.inner.query_turns(&q)?; + let enriched = filter_summary_enriched_turns( + enriched, + opts.agent.as_deref(), + agent_session_ids.as_ref(), + provider_filter.as_ref(), + ); + let turns = summary_turns_from_enriched(&enriched); + + match opts.mode { + SummaryReportMode::Grouped { by_provider } => { + let (group_by, tag_key, tag_values, rows) = if let Some(tag_key) = + opts.group_by_tag.as_deref() + { + let (rows, values) = summary_aggregate_by_tag(&enriched, tag_key, &pricing); + (SummaryGroupBy::Tag, Some(tag_key.to_string()), values, rows) + } else if by_provider { + ( + SummaryGroupBy::Provider, + None, + Vec::new(), + aggregate_by_provider(&turns, AggregateByProviderOptions::new(&pricing)) + .into_iter() + .map(summary_provider_to_aggregate_row) + .collect(), + ) + } else { + ( + SummaryGroupBy::Model, + None, + Vec::new(), + summary_aggregate_by_model(&turns, &pricing), + ) + }; + let total_cost = sum_costs(rows.iter().map(|r| &r.cost)); + let fidelity = summarize_fidelity(&turns); + let per_cell_fidelity = summary_per_cell_fidelity_to_value(&rows, group_by); + let replacement_savings = summarize_replacement_savings(&turns, None); + let quality = if opts.include_quality { + Some(compute_summary_quality_for_turns(&self.inner, &turns)?) + } else { + None + }; + let stop_reasons = StopReasonCounts::from_turns(&turns); + // Lazy walk over `~/.claude/projects/` (or the configured + // override) for the `subagents: X paired, Y orphan` + // summary line (issue #435). The walk short-circuits when + // the projects root is missing or every session lacks a + // `subagents/` subdir — i.e. zero cost on the vast + // majority of summaries that don't hit a session with + // sidecar transcripts. + // + // When the summary itself is scoped (any of `--session`, + // `--project`, `--since`, `--workflow`, `--tags`, + // `--agent`, `--providers`) we restrict the sidecar + // walk to the same session-id set the rest of the + // summary covers; otherwise the line could report + // paired/orphan counts from sessions the user excluded. + // Un-filtered runs keep the original global walk + // behavior. + let session_filter = summary_subagent_session_filter(&opts, &turns); + let subagents = compute_summary_subagent_counts(session_filter.as_ref()); + let (unpriced_turns, unpriced_models) = tally_unpriced(&turns, &pricing); + Ok(SummaryReport::Grouped(SummaryGroupedReport { + group_by, + tag_key, + tag_values, + turn_count: turns.len() as u64, + rows, + total_cost, + fidelity, + per_cell_fidelity, + replacement_savings, + stop_reasons, + subagents, + quality, + unpriced_turns, + unpriced_models, + })) + } + SummaryReportMode::ByTool => { + let attribution_turns = + load_summary_by_tool_attribution_turns(&self.inner, &enriched, &q)?; + let report = compute_summary_by_tool_report( + &self.inner, + &turns, + &attribution_turns, + &pricing, + )?; + Ok(SummaryReport::ByTool(report)) + } + SummaryReportMode::BySubagentType => { + let stats = + aggregate_subagent_type_stats(&turns, &BuildSubagentTreeOptions::new(&pricing)); + Ok(SummaryReport::BySubagentType(SummarySubagentTypeReport { + stats, + })) + } + SummaryReportMode::ByRelationship { subagent } => { + let relationships = self + .inner + .query_relationships(&summary_relationship_query_for_turn_slice(&q))?; + let matches = + match_summary_relationships_to_turns(&relationships, &turns, &pricing); + let stats = aggregate_summary_relationship_stats(&matches); + if subagent { + let subagent_types = aggregate_summary_relationship_subagent_stats(&matches); + let relationships = stats + .into_iter() + .filter(|s| s.relationship_type == RelationshipType::Subagent) + .collect(); + Ok(SummaryReport::Relationship(SummaryRelationshipReport { + relationships, + subagent_types, + })) + } else { + Ok(SummaryReport::Relationship(SummaryRelationshipReport { + relationships: stats, + subagent_types: Vec::new(), + })) + } + } + SummaryReportMode::SubagentTree { .. } => unreachable!(), + } + } +} + +pub fn summary_report(opts: SummaryReportOptions) -> Result { + let handle = open_with(opts.ledger_home.as_deref())?; + handle.summary_report(SummaryReportOptions { + ledger_home: None, + ..opts + }) +} + +pub fn summary_fidelity_summary_to_value(s: &FidelitySummary) -> serde_json::Value { + let mut by_class = serde_json::Map::new(); + for class in [ + FidelityClass::Full, + FidelityClass::UsageOnly, + FidelityClass::AggregateOnly, + FidelityClass::CostOnly, + FidelityClass::Partial, + ] { + by_class.insert( + class.wire_str().to_string(), + serde_json::json!(*s.by_class.get(&class).unwrap_or(&0)), + ); + } + + let mut by_granularity = serde_json::Map::new(); + for g in [ + UsageGranularity::PerTurn, + UsageGranularity::PerMessage, + UsageGranularity::PerSessionAggregate, + UsageGranularity::CostOnly, + ] { + by_granularity.insert( + g.wire_str().to_string(), + serde_json::json!(*s.by_granularity.get(&g).unwrap_or(&0)), + ); + } + + let mut missing = serde_json::Map::new(); + for field in [ + "hasInputTokens", + "hasOutputTokens", + "hasReasoningTokens", + "hasCacheReadTokens", + "hasCacheCreateTokens", + "hasToolCalls", + "hasToolResultEvents", + "hasSessionRelationships", + "hasRawContent", + ] { + missing.insert( + field.to_string(), + serde_json::json!(*s.missing_coverage.get(field).unwrap_or(&0)), + ); + } + + let mut out = serde_json::Map::new(); + out.insert("total".into(), serde_json::json!(s.total)); + out.insert("byClass".into(), serde_json::Value::Object(by_class)); + out.insert( + "byGranularity".into(), + serde_json::Value::Object(by_granularity), + ); + out.insert("missingCoverage".into(), serde_json::Value::Object(missing)); + out.insert("unknown".into(), serde_json::json!(s.unknown)); + serde_json::Value::Object(out) +} + +pub fn summary_per_cell_fidelity_to_value( + rows: &[UsageCostAggregateRow], + group_by: SummaryGroupBy, +) -> serde_json::Value { + let cells: Vec = rows + .iter() + .map(|r| { + let fields = [ + ("input", &r.coverage.input), + ("output", &r.coverage.output), + ("reasoning", &r.coverage.reasoning), + ("cacheRead", &r.coverage.cache_read), + ("cacheCreate", &r.coverage.cache_create), + ]; + let mut fields_map = serde_json::Map::new(); + let mut partial = false; + for (name, c) in fields { + if summary_cell_is_partial(c) || (c.known == 0 && c.missing > 0) { + partial = true; + } + fields_map.insert( + name.to_string(), + serde_json::json!({ + "known": c.known, + "missing": c.missing, + }), + ); + } + serde_json::json!({ + "label": r.label, + "partial": partial, + "fields": serde_json::Value::Object(fields_map), + }) + }) + .collect(); + serde_json::json!({ + "groupBy": group_by.wire_str(), + "cells": cells, + }) +} + +pub fn summary_replacement_savings_to_value( + savings: &ReplacementSavingsSummary, +) -> serde_json::Value { + let mut by_tool: Vec = savings + .by_tool + .iter() + .map(|(name, agg)| { + serde_json::json!({ + "tool": name, + "calls": agg.calls, + "collapsedCalls": agg.collapsed_calls, + "estimatedTokensSaved": agg.estimated_tokens_saved, + }) + }) + .collect(); + by_tool.sort_by(|a, b| { + let av = a + .get("estimatedTokensSaved") + .and_then(serde_json::Value::as_u64) + .unwrap_or(0); + let bv = b + .get("estimatedTokensSaved") + .and_then(serde_json::Value::as_u64) + .unwrap_or(0); + bv.cmp(&av).then_with(|| { + let at = a + .get("tool") + .and_then(serde_json::Value::as_str) + .unwrap_or(""); + let bt = b + .get("tool") + .and_then(serde_json::Value::as_str) + .unwrap_or(""); + at.cmp(bt) + }) + }); + serde_json::json!({ + "calls": savings.calls, + "collapsedCalls": savings.collapsed_calls, + "estimatedTokensSaved": savings.estimated_tokens_saved, + "byTool": by_tool, + }) +} + +mod compute; +pub(crate) use compute::*; diff --git a/crates/relayburn-sdk/src/reader/classifier.rs b/crates/relayburn-sdk/src/reader/classifier.rs index c7328b11..d3a8fb86 100644 --- a/crates/relayburn-sdk/src/reader/classifier.rs +++ b/crates/relayburn-sdk/src/reader/classifier.rs @@ -11,10 +11,18 @@ use std::sync::LazyLock; use phf::{phf_map, phf_set}; use regex::Regex; -use serde_json::{Map, Value}; use crate::reader::types::{ActivityCategory, ToolCall}; +mod bash_parse; +mod slash_triads; + +pub use bash_parse::{parse_bash_command, BashParse}; +// `SlashTriad` is part of the public `classifier::` surface even though no +// in-crate consumer names it yet; re-export it for reachability. +#[allow(unused_imports)] +pub use slash_triads::{detect_slash_triads, is_task_notification, SlashTriad}; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct ClassificationInput<'a> { pub tool_calls: &'a [ToolCall], @@ -30,13 +38,6 @@ pub struct ClassificationResult { pub has_edits: bool, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BashParse { - pub binary: String, - pub subcommand: Option, - pub normalized: String, -} - // --------------------------------------------------------------------------- // Static tool-name tables (phf — compile-time perfect hashing). // --------------------------------------------------------------------------- @@ -80,44 +81,11 @@ pub fn normalize_tool_name(name: &str) -> &str { TOOL_ALIASES.get(name).copied().unwrap_or(name) } -// --------------------------------------------------------------------------- -// Bash binary tables. -// --------------------------------------------------------------------------- - -static MULTIWORD_BINARIES: phf::Set<&'static str> = phf_set! { - "git", "gh", "npm", "pnpm", "yarn", "bun", "pip", "pip3", "uv", "poetry", - "cargo", "make", "docker", "kubectl", "helm", "terraform", "brew", "apt", - "apt-get", "gem", "bundle", "go", -}; - -static PACKAGE_RUNNERS: phf::Set<&'static str> = phf_set! { "npm", "pnpm", "yarn", "bun" }; -static SHELL_BINARIES: phf::Set<&'static str> = phf_set! { "bash", "sh", "zsh" }; -static PYTHON_BINARIES: phf::Set<&'static str> = phf_set! { "python", "python3" }; - -// `binary -> nested first-token subcommands`. Slices instead of a nested set -// because counts are tiny (1–6 entries) and linear scan is faster than a hash -// probe at that size. -static TWO_PART_SUBCOMMANDS: phf::Map<&'static str, &'static [&'static str]> = phf_map! { - "docker" => &["compose"], - "gh" => &["pr", "run", "issue", "repo", "workflow", "release"], - "go" => &["mod"], - "uv" => &["pip"], -}; - -static OPTION_TAKES_VALUE: phf::Set<&'static str> = phf_set! { - "-C", "-c", "-F", "--config", "--filter", "--git-dir", "--namespace", - "--prefix", "--repo", "--repository", "--work-tree", -}; - -static PYTHON_OPTION_TAKES_VALUE: phf::Set<&'static str> = phf_set! { - "-c", "-m", "-W", "-X", "--check-hash-based-pycs", -}; - // --------------------------------------------------------------------------- // Activity-keyword regexes (case-insensitive, word-boundary). // --------------------------------------------------------------------------- -fn build_re(s: &str) -> Regex { +pub(super) fn build_re(s: &str) -> Regex { Regex::new(s).expect("classifier regex failed to compile") } @@ -145,11 +113,6 @@ static BRAINSTORM_RE: LazyLock = LazyLock::new(|| { static PLANNING_RE: LazyLock = LazyLock::new(|| build_re(r"(?i)\b(plan(?:ning)?|outline|roadmap|strategy)\b")); -/// Matches a leading `KEY=` shell env-assignment token. Shared between -/// `skip_env_assignments` and `env_command_args` so the same compiled regex -/// is reused. -static ENV_ASSIGN_RE: LazyLock = LazyLock::new(|| build_re(r"^[A-Za-z_][A-Za-z0-9_]*=")); - // --------------------------------------------------------------------------- // BashRule — pattern plus optional `forbid` clause that emulates the TS // negative-lookahead idioms (e.g. `(?!.*--check\b)`). Encodes intent as data @@ -332,469 +295,6 @@ static DOC_FILE_RULES: LazyLock> = LazyLock::new(|| { .collect() }); -// --------------------------------------------------------------------------- -// Bash command parsing. -// --------------------------------------------------------------------------- - -pub fn parse_bash_command(command: &str) -> Option { - parse_bash_command_inner(command, 0) -} - -fn parse_bash_command_inner(command: &str, depth: u32) -> Option { - if depth > 5 { - return Some(shell_parse()); - } - let cmd = command.trim(); - if cmd.is_empty() { - return None; - } - if has_heredoc(cmd) || starts_with_compound_shell_syntax(cmd) { - return Some(shell_parse()); - } - if !has_balanced_shell_delimiters(cmd) { - return Some(shell_parse()); - } - - if let Some(unwrapped) = unwrap_subshell(cmd) { - return parse_bash_command_inner(&unwrapped, depth + 1); - } - if let Some(rest) = strip_leading_cd_prefix(cmd) { - return parse_bash_command_inner(&rest, depth + 1); - } - - let first = first_top_level_segment(cmd)?; - let cmd = first.trim(); - if cmd.is_empty() { - return None; - } - if has_heredoc(cmd) || starts_with_compound_shell_syntax(cmd) { - return Some(shell_parse()); - } - if let Some(unwrapped) = unwrap_subshell(cmd) { - return parse_bash_command_inner(&unwrapped, depth + 1); - } - - let tokens = match shell_words(cmd) { - Some(t) => t, - None => return Some(shell_parse()), - }; - let i = skip_env_assignments(&tokens, 0); - if i >= tokens.len() { - return None; - } - - let raw_binary = &tokens[i]; - let binary = normalize_binary(raw_binary); - - if SHELL_BINARIES.contains(binary.as_str()) { - if let Some(shell_cmd) = shell_command_arg(&tokens, i + 1) { - return parse_bash_command_inner(&shell_cmd, depth + 1); - } - return Some(verb(&binary, None)); - } - - if binary == "env" { - let env_args = env_command_args(&tokens, i + 1); - if env_args.is_empty() { - return None; - } - return parse_bash_command_inner(&env_args.join(" "), depth + 1); - } - - if PYTHON_BINARIES.contains(binary.as_str()) { - if let Some(parsed) = parse_python_module(&tokens, i + 1) { - return Some(parsed); - } - } - - if binary == "node" && tokens[i + 1..].iter().any(|t| t == "--test") { - return Some(verb(&binary, Some("--test"))); - } - - let sub_index = skip_leading_options(&tokens, i + 1); - if sub_index >= tokens.len() || !MULTIWORD_BINARIES.contains(binary.as_str()) { - return Some(verb(&binary, None)); - } - - let mut subcommand = tokens[sub_index].clone(); - if PACKAGE_RUNNERS.contains(binary.as_str()) - && subcommand == "run" - && sub_index + 1 < tokens.len() - { - subcommand = tokens[sub_index + 1].clone(); - } else if let Some(nested) = TWO_PART_SUBCOMMANDS.get(binary.as_str()) { - if nested.contains(&subcommand.as_str()) && sub_index + 1 < tokens.len() { - subcommand = format!("{} {}", subcommand, tokens[sub_index + 1]); - } - } - - Some(verb(&binary, Some(&subcommand))) -} - -fn verb(binary: &str, subcommand: Option<&str>) -> BashParse { - let normalized = match subcommand { - Some(sub) => format!("{binary} {sub}"), - None => binary.to_string(), - }; - BashParse { - binary: binary.to_string(), - subcommand: subcommand.map(str::to_string), - normalized, - } -} - -fn shell_parse() -> BashParse { - BashParse { - binary: "(shell)".to_string(), - subcommand: None, - normalized: "(shell)".to_string(), - } -} - -fn has_heredoc(cmd: &str) -> bool { - static RE: LazyLock = LazyLock::new(|| build_re(r"<<-?\s*\S+")); - RE.is_match(cmd) -} - -fn starts_with_compound_shell_syntax(cmd: &str) -> bool { - static RE: LazyLock = - LazyLock::new(|| build_re(r"^(?:for|while|until|if|case|select|function)\b")); - static BRACE_RE: LazyLock = LazyLock::new(|| build_re(r"^\{\s")); - RE.is_match(cmd) || BRACE_RE.is_match(cmd) -} - -fn has_balanced_shell_delimiters(cmd: &str) -> bool { - let mut quote: Option = None; - let mut escaped = false; - let mut depth: i32 = 0; - for &b in cmd.as_bytes() { - if escaped { - escaped = false; - continue; - } - if b == b'\\' && quote != Some(b'\'') { - escaped = true; - continue; - } - if let Some(q) = quote { - if b == q { - quote = None; - } - continue; - } - if b == b'"' || b == b'\'' { - quote = Some(b); - continue; - } - if b == b'(' { - depth += 1; - } else if b == b')' { - depth -= 1; - if depth < 0 { - return false; - } - } - } - quote.is_none() && depth == 0 -} - -fn unwrap_subshell(cmd: &str) -> Option { - if !cmd.starts_with('(') || !cmd.ends_with(')') { - return None; - } - let bytes = cmd.as_bytes(); - let mut quote: Option = None; - let mut escaped = false; - let mut depth: i32 = 0; - for (i, &b) in bytes.iter().enumerate() { - if escaped { - escaped = false; - continue; - } - if b == b'\\' && quote != Some(b'\'') { - escaped = true; - continue; - } - if let Some(q) = quote { - if b == q { - quote = None; - } - continue; - } - if b == b'"' || b == b'\'' { - quote = Some(b); - continue; - } - if b == b'(' { - depth += 1; - } else if b == b')' { - depth -= 1; - if depth == 0 && i != bytes.len() - 1 { - return None; - } - if depth < 0 { - return None; - } - } - } - if quote.is_some() || depth != 0 { - return None; - } - Some(cmd[1..cmd.len() - 1].trim().to_string()) -} - -fn strip_leading_cd_prefix(cmd: &str) -> Option { - let op = first_top_level_operator(cmd, &["&&", ";"])?; - let words = shell_words(cmd[..op.index].trim())?; - let cmd_idx = skip_env_assignments(&words, 0); - if words.get(cmd_idx).map(String::as_str) != Some("cd") { - return None; - } - Some(cmd[op.index + op.operator.len()..].trim().to_string()) -} - -fn first_top_level_segment(cmd: &str) -> Option { - match first_top_level_operator(cmd, &["&&", "||", ";", "|", "\n"]) { - Some(op) => Some(cmd[..op.index].to_string()), - None => Some(cmd.to_string()), - } -} - -struct OpHit { - index: usize, - operator: &'static str, -} - -fn first_top_level_operator(cmd: &str, operators: &[&'static str]) -> Option { - let bytes = cmd.as_bytes(); - let mut quote: Option = None; - let mut escaped = false; - let mut depth: i32 = 0; - let mut i = 0; - while i < bytes.len() { - let b = bytes[i]; - if escaped { - escaped = false; - i += 1; - continue; - } - if b == b'\\' && quote != Some(b'\'') { - escaped = true; - i += 1; - continue; - } - if let Some(q) = quote { - if b == q { - quote = None; - } - i += 1; - continue; - } - if b == b'"' || b == b'\'' { - quote = Some(b); - i += 1; - continue; - } - if b == b'(' { - depth += 1; - i += 1; - continue; - } - if b == b')' { - depth -= 1; - if depth < 0 { - return None; - } - i += 1; - continue; - } - if depth == 0 { - for op in operators { - // Compare against the byte slice rather than `cmd[i..]` — - // operators are all ASCII, and `i` may sit inside a multi-byte - // UTF-8 sequence (e.g. when `cmd` contains a path like - // `café.txt`), where `&str` slicing would panic. - if bytes[i..].starts_with(op.as_bytes()) { - return Some(OpHit { - index: i, - operator: op, - }); - } - } - } - i += 1; - } - None -} - -fn shell_words(segment: &str) -> Option> { - let mut words = Vec::new(); - let mut quote: Option = None; - let mut escaped = false; - let mut current = String::new(); - for ch in segment.chars() { - if escaped { - current.push(ch); - escaped = false; - continue; - } - if ch == '\\' && quote != Some('\'') { - escaped = true; - continue; - } - if let Some(q) = quote { - if ch == q { - quote = None; - } else { - current.push(ch); - } - continue; - } - if ch == '"' || ch == '\'' { - quote = Some(ch); - continue; - } - if ch.is_whitespace() { - if !current.is_empty() { - words.push(std::mem::take(&mut current)); - } - continue; - } - current.push(ch); - } - if escaped { - current.push('\\'); - } - if quote.is_some() { - return None; - } - if !current.is_empty() { - words.push(current); - } - Some(words) -} - -fn skip_env_assignments(tokens: &[String], start: usize) -> usize { - let mut i = start; - while i < tokens.len() && ENV_ASSIGN_RE.is_match(&tokens[i]) { - i += 1; - } - i -} - -fn skip_leading_options(tokens: &[String], start: usize) -> usize { - let mut i = start; - while i < tokens.len() { - let token = &tokens[i]; - if token == "--" { - return i + 1; - } - if !token.starts_with('-') { - return i; - } - let has_eq = token.contains('='); - let option_name = if has_eq { - token.split('=').next().unwrap() - } else { - token.as_str() - }; - i += 1; - if !has_eq && OPTION_TAKES_VALUE.contains(option_name) && i < tokens.len() { - i += 1; - } - } - i -} - -fn shell_command_arg(tokens: &[String], start: usize) -> Option { - for (offset, token) in tokens[start..].iter().enumerate() { - let i = start + offset; - if token == "-c" - || (token.starts_with('-') && !token.starts_with("--") && token.contains('c')) - { - return tokens.get(i + 1).cloned(); - } - } - None -} - -fn env_command_args(tokens: &[String], start: usize) -> Vec { - let mut i = start; - while i < tokens.len() { - let token = &tokens[i]; - if token == "--" { - i += 1; - break; - } - if ENV_ASSIGN_RE.is_match(token) { - i += 1; - continue; - } - if token.starts_with('-') { - i += 1; - continue; - } - break; - } - tokens[i..].to_vec() -} - -fn parse_python_module(tokens: &[String], start: usize) -> Option { - let module_flag = find_python_module_flag(tokens, start)?; - if module_flag + 1 >= tokens.len() { - return None; - } - let module = normalize_binary(&tokens[module_flag + 1]); - if module == "pytest" { - return Some(verb("pytest", None)); - } - if module == "pip" || module == "pip3" { - let sub_index = skip_leading_options(tokens, module_flag + 2); - if sub_index < tokens.len() { - return Some(verb(&module, Some(&tokens[sub_index]))); - } - return Some(verb(&module, None)); - } - Some(verb(&module, None)) -} - -fn find_python_module_flag(tokens: &[String], start: usize) -> Option { - let mut i = start; - while i < tokens.len() { - let token = &tokens[i]; - if token == "--" { - return None; - } - if token == "-m" { - return Some(i); - } - if token == "-c" { - return None; - } - if !token.starts_with('-') || token == "-" { - return None; - } - let has_eq = token.contains('='); - let option_name = if has_eq { - token.split('=').next().unwrap() - } else { - token.as_str() - }; - if !has_eq && PYTHON_OPTION_TAKES_VALUE.contains(option_name) { - i += 1; - } - i += 1; - } - None -} - -fn normalize_binary(raw: &str) -> String { - raw.split('/') - .rfind(|s| !s.is_empty()) - .unwrap_or(raw) - .to_string() -} - // --------------------------------------------------------------------------- // classify_activity priority ladder. // --------------------------------------------------------------------------- @@ -998,334 +498,13 @@ pub fn count_retries(tool_calls: &[ToolCall]) -> u64 { retries } -// --------------------------------------------------------------------------- -// Harness-injected `` row detector. -// -// Claude Code's harness writes synthetic rows when a background Bash task -// finishes. They share the envelope shape of queued user prompts, so a -// text-prefix-only filter (just checking for `` in -// content) would false-positive on a real user prompt that legitimately -// types that string. Each clause AND-checks shape AND purpose so -// legitimate prompts with the same shape but a different `commandMode` / -// `origin.kind` survive. -// --------------------------------------------------------------------------- - -/// True when a raw JSONL row represents a harness-injected -/// `` synthetic message rather than a real user prompt. -/// Three clauses, each ANDing a shape check with a purpose check: -/// -/// 1. `type == "queue-operation"` AND `content` is a string starting with -/// ``. -/// 2. `origin.kind == "task-notification"`. -/// 3. `attachment.type == "queued_command"` AND -/// `attachment.commandMode == "task-notification"`. -pub fn is_task_notification(row: &Map) -> bool { - // Clause 1: queue-operation row whose top-level content starts with the - // `` marker. - if row.get("type").and_then(Value::as_str) == Some("queue-operation") - && row - .get("content") - .and_then(Value::as_str) - .is_some_and(|s| s.starts_with("")) - { - return true; - } - // Clause 2: explicit `origin.kind` marker. The shape check is implicit - // in dereferencing `origin` as an object; the purpose check is the - // `task-notification` string match. - if row - .get("origin") - .and_then(Value::as_object) - .and_then(|o| o.get("kind")) - .and_then(Value::as_str) - == Some("task-notification") - { - return true; - } - // Clause 3: queued-command attachment whose `commandMode` is - // `task-notification`. Both fields must match — a legitimate queued - // user prompt has `attachment.type == "queued_command"` but a different - // `commandMode`, and must survive this check. - if let Some(att) = row.get("attachment").and_then(Value::as_object) { - let ty = att.get("type").and_then(Value::as_str); - let mode = att.get("commandMode").and_then(Value::as_str); - if ty == Some("queued_command") && mode == Some("task-notification") { - return true; - } - } - false -} - -// --------------------------------------------------------------------------- -// Slash-command triad detector — see #438. -// -// Claude Code's slash commands (`/review`, `/init`, custom skills) emit a -// deterministic three-row sequence in the JSONL transcript: -// -// caveat — synthetic user row introducing the command. Body opens -// with the literal `Caveat:` prefix and the row is the -// apparent root of the parent chain for the next two -// rows. Carries no real user intent — it's a harness -// artifact. -// invocation — user row whose `parentUuid == caveat.uuid`. Body -// carries the synthetic command envelope: -// ``, ``, optionally -// ``. -// stdout — user row whose `parentUuid == invocation.uuid`. Body -// carries the captured stdout in a `` -// block. -// -// The classifier historically treated each row as a separate activity, -// which trebled the apparent activity count for sessions that lean on -// slash commands. The detector here collapses the triad into one -// synthetic `Skill` activity for downstream rollups; token attribution -// stays on the underlying rows so `burn hotspots` isn't double-charged. -// -// Pinning detection on parent-UUID chain shape (not on the exact text -// prefix of the caveat row) is deliberate: the literal `Caveat:` opener -// has drifted across Claude Code versions, but the chain shape — three -// rows linked caveat → invocation → stdout — has stayed stable. We use -// the text markers `` and `` as -// purpose checks on the invocation and stdout rows (so an unrelated -// three-row chain that happens to look structurally similar — e.g. a -// real user prompt followed by an assistant reply followed by a tool -// result — does NOT misdetect), but the chain-shape predicate carries -// the primary signal. See `is_task_notification` (#442) for the same -// shape-AND-purpose pattern. - -const CAVEAT_PREFIX: &str = "Caveat:"; -const COMMAND_NAME_OPEN: &str = ""; -const COMMAND_NAME_CLOSE: &str = ""; -const LOCAL_STDOUT_OPEN: &str = ""; - -/// One detected slash-command triad: the three row indices into the -/// original `rows` slice and the extracted skill name (when the -/// invocation body exposed a `` block; `None` otherwise). -/// -/// Indices are stable references into the caller's input — the -/// downstream wiring in the Claude reader uses them to override per-row -/// activity attribution without re-walking the input. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SlashTriad { - pub caveat_idx: usize, - pub invocation_idx: usize, - pub stdout_idx: usize, - pub skill_name: Option, -} - -/// Detect Claude slash-command triads in a flat slice of raw JSONL rows -/// (already-parsed as `serde_json::Map`s). -/// -/// The walk is O(n): build a `uuid -> index` index once, then for each -/// candidate caveat row look up the two children by their UUID chain. -/// Each row is consumed by at most one triad — the row-index sets are -/// disjoint by construction (we mark each used index in a bitset). -/// -/// Caller obligations: -/// -/// - The slice must preserve JSONL emission order. The detector does -/// not sort or filter — it inspects rows in place. -/// - Rows are matched on the parent-UUID chain shape; text checks on -/// the invocation and stdout rows are purpose guards that block -/// structurally-similar but semantically-different chains (e.g. a -/// normal user → assistant → tool_result chain) from misdetecting. -pub fn detect_slash_triads(rows: &[Map]) -> Vec { - if rows.len() < 3 { - return Vec::new(); - } - // Track rows already consumed by a prior triad so a single row can't - // be the invocation of triad A and the caveat of triad B simultaneously. - let mut consumed = vec![false; rows.len()]; - let mut out: Vec = Vec::new(); - - for (caveat_idx, caveat) in rows.iter().enumerate() { - if consumed[caveat_idx] { - continue; - } - if !is_caveat_row(caveat) { - continue; - } - let caveat_uuid = match caveat.get("uuid").and_then(Value::as_str) { - Some(u) if !u.is_empty() => u, - _ => continue, - }; - // The invocation row must (a) point at the caveat via parentUuid - // AND (b) carry an invocation body. The latter is the purpose - // check that blocks an unrelated child of the caveat from - // promoting the chain into a Skill. - let (invocation_idx, invocation) = match find_first_unconsumed_child( - rows, - &consumed, - caveat_uuid, - caveat_idx + 1, - is_invocation_row, - ) { - Some(pair) => pair, - None => continue, - }; - let invocation_uuid = match invocation.get("uuid").and_then(Value::as_str) { - Some(u) if !u.is_empty() => u, - _ => continue, - }; - let (stdout_idx, _stdout) = match find_first_unconsumed_child( - rows, - &consumed, - invocation_uuid, - invocation_idx + 1, - is_stdout_row, - ) { - Some(pair) => pair, - None => continue, - }; - let skill_name = extract_skill_name(invocation).or_else(|| extract_skill_name(caveat)); - consumed[caveat_idx] = true; - consumed[invocation_idx] = true; - consumed[stdout_idx] = true; - out.push(SlashTriad { - caveat_idx, - invocation_idx, - stdout_idx, - skill_name, - }); - } - out -} - -/// True when this row matches the caveat shape: a user-typed row whose -/// extracted text body begins with the `Caveat:` literal. We deliberately -/// keep this lightweight — the structural check (caveat is the root of -/// the chain a downstream invocation/stdout points at) carries the -/// primary signal. -fn is_caveat_row(row: &Map) -> bool { - if row.get("type").and_then(Value::as_str) != Some("user") { - return false; - } - extract_row_text(row) - .map(|s| s.trim_start().starts_with(CAVEAT_PREFIX)) - .unwrap_or(false) -} - -/// True when this row carries a `` block — the invocation -/// payload of a slash command. -fn is_invocation_row(row: &Map) -> bool { - if row.get("type").and_then(Value::as_str) != Some("user") { - return false; - } - extract_row_text(row) - .map(|s| s.contains(COMMAND_NAME_OPEN)) - .unwrap_or(false) -} - -/// True when this row carries a `` block — the -/// stdout-capture payload of a slash command. -fn is_stdout_row(row: &Map) -> bool { - if row.get("type").and_then(Value::as_str) != Some("user") { - return false; - } - extract_row_text(row) - .map(|s| s.contains(LOCAL_STDOUT_OPEN)) - .unwrap_or(false) -} - -/// Walk forward from `start_idx` looking for the first not-yet-consumed -/// row whose `parentUuid == parent_uuid` and which passes the purpose -/// `check`. Returns `(idx, &row)` or `None`. Forward-only because the -/// JSONL ordering for these rows is stable in practice (the harness -/// writes caveat → invocation → stdout adjacently); a sibling reorder -/// would land both the invocation and stdout *after* the caveat. -fn find_first_unconsumed_child<'a, F>( - rows: &'a [Map], - consumed: &[bool], - parent_uuid: &str, - start_idx: usize, - check: F, -) -> Option<(usize, &'a Map)> -where - F: Fn(&Map) -> bool, -{ - for (offset, row) in rows[start_idx..].iter().enumerate() { - let idx = start_idx + offset; - if consumed[idx] { - continue; - } - let pu = row.get("parentUuid").and_then(Value::as_str).unwrap_or(""); - if pu != parent_uuid { - continue; - } - if check(row) { - return Some((idx, row)); - } - } - None -} - -/// Pull `...` text from either the caveat -/// or invocation row. Returns `None` if no marker block is present. -fn extract_skill_name(row: &Map) -> Option { - let text = extract_row_text(row)?; - let open = text.find(COMMAND_NAME_OPEN)?; - let after = &text[open + COMMAND_NAME_OPEN.len()..]; - let close = after.find(COMMAND_NAME_CLOSE)?; - let raw = after[..close].trim(); - if raw.is_empty() { - return None; - } - // Tolerate the optional leading `/` (matches the ghost_surface - // miner's accept-with-or-without convention). - Some(raw.trim_start_matches('/').to_string()) -} - -/// Extract the row's plain text body (string content, or concatenation -/// of `text` / `content` strings inside an array body). Mirrors -/// `extract_plain_user_text_from_obj` from the Claude reader closely -/// enough to detect markers; we read the `content` field of a user-typed -/// row whose body may be a string or a list of blocks. -fn extract_row_text(row: &Map) -> Option { - let body = row.get("message").and_then(|m| m.get("content"))?; - if let Some(s) = body.as_str() { - if s.is_empty() { - return None; - } - return Some(s.to_string()); - } - let arr = body.as_array()?; - let mut parts: Vec = Vec::new(); - for block in arr { - let bo = match block.as_object() { - Some(o) => o, - None => continue, - }; - // `{"type":"text","text":"..."}` — assistant-style text block. - if bo.get("type").and_then(Value::as_str) == Some("text") { - if let Some(s) = bo.get("text").and_then(Value::as_str) { - if !s.is_empty() { - parts.push(s.to_string()); - } - } - continue; - } - // `{"type":"tool_result","content":"..."}` — string-content tool - // result envelope. The slash-command stdout row can ship its - // payload this way; we still want the marker visible to the - // purpose check. - if let Some(s) = bo.get("content").and_then(Value::as_str) { - if !s.is_empty() { - parts.push(s.to_string()); - } - } - } - if parts.is_empty() { - None - } else { - Some(parts.join("\n")) - } -} - // --------------------------------------------------------------------------- // Tests. // --------------------------------------------------------------------------- #[cfg(test)] mod tests { + use serde_json::{Map, Value}; + use super::*; fn tc(name: &str, target: Option<&str>) -> ToolCall { diff --git a/crates/relayburn-sdk/src/reader/classifier/bash_parse.rs b/crates/relayburn-sdk/src/reader/classifier/bash_parse.rs new file mode 100644 index 00000000..5301a47d --- /dev/null +++ b/crates/relayburn-sdk/src/reader/classifier/bash_parse.rs @@ -0,0 +1,516 @@ +//! Bash-command parser — extracted from the classifier. + +use std::sync::LazyLock; + +use phf::{phf_map, phf_set}; +use regex::Regex; + +use super::build_re; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BashParse { + pub binary: String, + pub subcommand: Option, + pub normalized: String, +} + +// --------------------------------------------------------------------------- +// Bash binary tables. +// --------------------------------------------------------------------------- + +static MULTIWORD_BINARIES: phf::Set<&'static str> = phf_set! { + "git", "gh", "npm", "pnpm", "yarn", "bun", "pip", "pip3", "uv", "poetry", + "cargo", "make", "docker", "kubectl", "helm", "terraform", "brew", "apt", + "apt-get", "gem", "bundle", "go", +}; + +static PACKAGE_RUNNERS: phf::Set<&'static str> = phf_set! { "npm", "pnpm", "yarn", "bun" }; +static SHELL_BINARIES: phf::Set<&'static str> = phf_set! { "bash", "sh", "zsh" }; +static PYTHON_BINARIES: phf::Set<&'static str> = phf_set! { "python", "python3" }; + +// `binary -> nested first-token subcommands`. Slices instead of a nested set +// because counts are tiny (1–6 entries) and linear scan is faster than a hash +// probe at that size. +static TWO_PART_SUBCOMMANDS: phf::Map<&'static str, &'static [&'static str]> = phf_map! { + "docker" => &["compose"], + "gh" => &["pr", "run", "issue", "repo", "workflow", "release"], + "go" => &["mod"], + "uv" => &["pip"], +}; + +static OPTION_TAKES_VALUE: phf::Set<&'static str> = phf_set! { + "-C", "-c", "-F", "--config", "--filter", "--git-dir", "--namespace", + "--prefix", "--repo", "--repository", "--work-tree", +}; + +static PYTHON_OPTION_TAKES_VALUE: phf::Set<&'static str> = phf_set! { + "-c", "-m", "-W", "-X", "--check-hash-based-pycs", +}; + +/// Matches a leading `KEY=` shell env-assignment token. Shared between +/// `skip_env_assignments` and `env_command_args` so the same compiled regex +/// is reused. +static ENV_ASSIGN_RE: LazyLock = LazyLock::new(|| build_re(r"^[A-Za-z_][A-Za-z0-9_]*=")); + +// --------------------------------------------------------------------------- +// Bash command parsing. +// --------------------------------------------------------------------------- + +pub fn parse_bash_command(command: &str) -> Option { + parse_bash_command_inner(command, 0) +} + +fn parse_bash_command_inner(command: &str, depth: u32) -> Option { + if depth > 5 { + return Some(shell_parse()); + } + let cmd = command.trim(); + if cmd.is_empty() { + return None; + } + if has_heredoc(cmd) || starts_with_compound_shell_syntax(cmd) { + return Some(shell_parse()); + } + if !has_balanced_shell_delimiters(cmd) { + return Some(shell_parse()); + } + + if let Some(unwrapped) = unwrap_subshell(cmd) { + return parse_bash_command_inner(&unwrapped, depth + 1); + } + if let Some(rest) = strip_leading_cd_prefix(cmd) { + return parse_bash_command_inner(&rest, depth + 1); + } + + let first = first_top_level_segment(cmd)?; + let cmd = first.trim(); + if cmd.is_empty() { + return None; + } + if has_heredoc(cmd) || starts_with_compound_shell_syntax(cmd) { + return Some(shell_parse()); + } + if let Some(unwrapped) = unwrap_subshell(cmd) { + return parse_bash_command_inner(&unwrapped, depth + 1); + } + + let tokens = match shell_words(cmd) { + Some(t) => t, + None => return Some(shell_parse()), + }; + let i = skip_env_assignments(&tokens, 0); + if i >= tokens.len() { + return None; + } + + let raw_binary = &tokens[i]; + let binary = normalize_binary(raw_binary); + + if SHELL_BINARIES.contains(binary.as_str()) { + if let Some(shell_cmd) = shell_command_arg(&tokens, i + 1) { + return parse_bash_command_inner(&shell_cmd, depth + 1); + } + return Some(verb(&binary, None)); + } + + if binary == "env" { + let env_args = env_command_args(&tokens, i + 1); + if env_args.is_empty() { + return None; + } + return parse_bash_command_inner(&env_args.join(" "), depth + 1); + } + + if PYTHON_BINARIES.contains(binary.as_str()) { + if let Some(parsed) = parse_python_module(&tokens, i + 1) { + return Some(parsed); + } + } + + if binary == "node" && tokens[i + 1..].iter().any(|t| t == "--test") { + return Some(verb(&binary, Some("--test"))); + } + + let sub_index = skip_leading_options(&tokens, i + 1); + if sub_index >= tokens.len() || !MULTIWORD_BINARIES.contains(binary.as_str()) { + return Some(verb(&binary, None)); + } + + let mut subcommand = tokens[sub_index].clone(); + if PACKAGE_RUNNERS.contains(binary.as_str()) + && subcommand == "run" + && sub_index + 1 < tokens.len() + { + subcommand = tokens[sub_index + 1].clone(); + } else if let Some(nested) = TWO_PART_SUBCOMMANDS.get(binary.as_str()) { + if nested.contains(&subcommand.as_str()) && sub_index + 1 < tokens.len() { + subcommand = format!("{} {}", subcommand, tokens[sub_index + 1]); + } + } + + Some(verb(&binary, Some(&subcommand))) +} + +fn verb(binary: &str, subcommand: Option<&str>) -> BashParse { + let normalized = match subcommand { + Some(sub) => format!("{binary} {sub}"), + None => binary.to_string(), + }; + BashParse { + binary: binary.to_string(), + subcommand: subcommand.map(str::to_string), + normalized, + } +} + +fn shell_parse() -> BashParse { + BashParse { + binary: "(shell)".to_string(), + subcommand: None, + normalized: "(shell)".to_string(), + } +} + +fn has_heredoc(cmd: &str) -> bool { + static RE: LazyLock = LazyLock::new(|| build_re(r"<<-?\s*\S+")); + RE.is_match(cmd) +} + +fn starts_with_compound_shell_syntax(cmd: &str) -> bool { + static RE: LazyLock = + LazyLock::new(|| build_re(r"^(?:for|while|until|if|case|select|function)\b")); + static BRACE_RE: LazyLock = LazyLock::new(|| build_re(r"^\{\s")); + RE.is_match(cmd) || BRACE_RE.is_match(cmd) +} + +fn has_balanced_shell_delimiters(cmd: &str) -> bool { + let mut quote: Option = None; + let mut escaped = false; + let mut depth: i32 = 0; + for &b in cmd.as_bytes() { + if escaped { + escaped = false; + continue; + } + if b == b'\\' && quote != Some(b'\'') { + escaped = true; + continue; + } + if let Some(q) = quote { + if b == q { + quote = None; + } + continue; + } + if b == b'"' || b == b'\'' { + quote = Some(b); + continue; + } + if b == b'(' { + depth += 1; + } else if b == b')' { + depth -= 1; + if depth < 0 { + return false; + } + } + } + quote.is_none() && depth == 0 +} + +fn unwrap_subshell(cmd: &str) -> Option { + if !cmd.starts_with('(') || !cmd.ends_with(')') { + return None; + } + let bytes = cmd.as_bytes(); + let mut quote: Option = None; + let mut escaped = false; + let mut depth: i32 = 0; + for (i, &b) in bytes.iter().enumerate() { + if escaped { + escaped = false; + continue; + } + if b == b'\\' && quote != Some(b'\'') { + escaped = true; + continue; + } + if let Some(q) = quote { + if b == q { + quote = None; + } + continue; + } + if b == b'"' || b == b'\'' { + quote = Some(b); + continue; + } + if b == b'(' { + depth += 1; + } else if b == b')' { + depth -= 1; + if depth == 0 && i != bytes.len() - 1 { + return None; + } + if depth < 0 { + return None; + } + } + } + if quote.is_some() || depth != 0 { + return None; + } + Some(cmd[1..cmd.len() - 1].trim().to_string()) +} + +fn strip_leading_cd_prefix(cmd: &str) -> Option { + let op = first_top_level_operator(cmd, &["&&", ";"])?; + let words = shell_words(cmd[..op.index].trim())?; + let cmd_idx = skip_env_assignments(&words, 0); + if words.get(cmd_idx).map(String::as_str) != Some("cd") { + return None; + } + Some(cmd[op.index + op.operator.len()..].trim().to_string()) +} + +fn first_top_level_segment(cmd: &str) -> Option { + match first_top_level_operator(cmd, &["&&", "||", ";", "|", "\n"]) { + Some(op) => Some(cmd[..op.index].to_string()), + None => Some(cmd.to_string()), + } +} + +struct OpHit { + index: usize, + operator: &'static str, +} + +fn first_top_level_operator(cmd: &str, operators: &[&'static str]) -> Option { + let bytes = cmd.as_bytes(); + let mut quote: Option = None; + let mut escaped = false; + let mut depth: i32 = 0; + let mut i = 0; + while i < bytes.len() { + let b = bytes[i]; + if escaped { + escaped = false; + i += 1; + continue; + } + if b == b'\\' && quote != Some(b'\'') { + escaped = true; + i += 1; + continue; + } + if let Some(q) = quote { + if b == q { + quote = None; + } + i += 1; + continue; + } + if b == b'"' || b == b'\'' { + quote = Some(b); + i += 1; + continue; + } + if b == b'(' { + depth += 1; + i += 1; + continue; + } + if b == b')' { + depth -= 1; + if depth < 0 { + return None; + } + i += 1; + continue; + } + if depth == 0 { + for op in operators { + // Compare against the byte slice rather than `cmd[i..]` — + // operators are all ASCII, and `i` may sit inside a multi-byte + // UTF-8 sequence (e.g. when `cmd` contains a path like + // `café.txt`), where `&str` slicing would panic. + if bytes[i..].starts_with(op.as_bytes()) { + return Some(OpHit { + index: i, + operator: op, + }); + } + } + } + i += 1; + } + None +} + +fn shell_words(segment: &str) -> Option> { + let mut words = Vec::new(); + let mut quote: Option = None; + let mut escaped = false; + let mut current = String::new(); + for ch in segment.chars() { + if escaped { + current.push(ch); + escaped = false; + continue; + } + if ch == '\\' && quote != Some('\'') { + escaped = true; + continue; + } + if let Some(q) = quote { + if ch == q { + quote = None; + } else { + current.push(ch); + } + continue; + } + if ch == '"' || ch == '\'' { + quote = Some(ch); + continue; + } + if ch.is_whitespace() { + if !current.is_empty() { + words.push(std::mem::take(&mut current)); + } + continue; + } + current.push(ch); + } + if escaped { + current.push('\\'); + } + if quote.is_some() { + return None; + } + if !current.is_empty() { + words.push(current); + } + Some(words) +} + +fn skip_env_assignments(tokens: &[String], start: usize) -> usize { + let mut i = start; + while i < tokens.len() && ENV_ASSIGN_RE.is_match(&tokens[i]) { + i += 1; + } + i +} + +fn skip_leading_options(tokens: &[String], start: usize) -> usize { + let mut i = start; + while i < tokens.len() { + let token = &tokens[i]; + if token == "--" { + return i + 1; + } + if !token.starts_with('-') { + return i; + } + let has_eq = token.contains('='); + let option_name = if has_eq { + token.split('=').next().unwrap() + } else { + token.as_str() + }; + i += 1; + if !has_eq && OPTION_TAKES_VALUE.contains(option_name) && i < tokens.len() { + i += 1; + } + } + i +} + +fn shell_command_arg(tokens: &[String], start: usize) -> Option { + for (offset, token) in tokens[start..].iter().enumerate() { + let i = start + offset; + if token == "-c" + || (token.starts_with('-') && !token.starts_with("--") && token.contains('c')) + { + return tokens.get(i + 1).cloned(); + } + } + None +} + +fn env_command_args(tokens: &[String], start: usize) -> Vec { + let mut i = start; + while i < tokens.len() { + let token = &tokens[i]; + if token == "--" { + i += 1; + break; + } + if ENV_ASSIGN_RE.is_match(token) { + i += 1; + continue; + } + if token.starts_with('-') { + i += 1; + continue; + } + break; + } + tokens[i..].to_vec() +} + +fn parse_python_module(tokens: &[String], start: usize) -> Option { + let module_flag = find_python_module_flag(tokens, start)?; + if module_flag + 1 >= tokens.len() { + return None; + } + let module = normalize_binary(&tokens[module_flag + 1]); + if module == "pytest" { + return Some(verb("pytest", None)); + } + if module == "pip" || module == "pip3" { + let sub_index = skip_leading_options(tokens, module_flag + 2); + if sub_index < tokens.len() { + return Some(verb(&module, Some(&tokens[sub_index]))); + } + return Some(verb(&module, None)); + } + Some(verb(&module, None)) +} + +fn find_python_module_flag(tokens: &[String], start: usize) -> Option { + let mut i = start; + while i < tokens.len() { + let token = &tokens[i]; + if token == "--" { + return None; + } + if token == "-m" { + return Some(i); + } + if token == "-c" { + return None; + } + if !token.starts_with('-') || token == "-" { + return None; + } + let has_eq = token.contains('='); + let option_name = if has_eq { + token.split('=').next().unwrap() + } else { + token.as_str() + }; + if !has_eq && PYTHON_OPTION_TAKES_VALUE.contains(option_name) { + i += 1; + } + i += 1; + } + None +} + +fn normalize_binary(raw: &str) -> String { + raw.split('/') + .rfind(|s| !s.is_empty()) + .unwrap_or(raw) + .to_string() +} diff --git a/crates/relayburn-sdk/src/reader/classifier/slash_triads.rs b/crates/relayburn-sdk/src/reader/classifier/slash_triads.rs new file mode 100644 index 00000000..782456cd --- /dev/null +++ b/crates/relayburn-sdk/src/reader/classifier/slash_triads.rs @@ -0,0 +1,327 @@ +//! Slash-command triad + task-notification row detection — extracted from the +//! classifier. + +use serde_json::{Map, Value}; + +// --------------------------------------------------------------------------- +// Harness-injected `` row detector. +// +// Claude Code's harness writes synthetic rows when a background Bash task +// finishes. They share the envelope shape of queued user prompts, so a +// text-prefix-only filter (just checking for `` in +// content) would false-positive on a real user prompt that legitimately +// types that string. Each clause AND-checks shape AND purpose so +// legitimate prompts with the same shape but a different `commandMode` / +// `origin.kind` survive. +// --------------------------------------------------------------------------- + +/// True when a raw JSONL row represents a harness-injected +/// `` synthetic message rather than a real user prompt. +/// Three clauses, each ANDing a shape check with a purpose check: +/// +/// 1. `type == "queue-operation"` AND `content` is a string starting with +/// ``. +/// 2. `origin.kind == "task-notification"`. +/// 3. `attachment.type == "queued_command"` AND +/// `attachment.commandMode == "task-notification"`. +pub fn is_task_notification(row: &Map) -> bool { + // Clause 1: queue-operation row whose top-level content starts with the + // `` marker. + if row.get("type").and_then(Value::as_str) == Some("queue-operation") + && row + .get("content") + .and_then(Value::as_str) + .is_some_and(|s| s.starts_with("")) + { + return true; + } + // Clause 2: explicit `origin.kind` marker. The shape check is implicit + // in dereferencing `origin` as an object; the purpose check is the + // `task-notification` string match. + if row + .get("origin") + .and_then(Value::as_object) + .and_then(|o| o.get("kind")) + .and_then(Value::as_str) + == Some("task-notification") + { + return true; + } + // Clause 3: queued-command attachment whose `commandMode` is + // `task-notification`. Both fields must match — a legitimate queued + // user prompt has `attachment.type == "queued_command"` but a different + // `commandMode`, and must survive this check. + if let Some(att) = row.get("attachment").and_then(Value::as_object) { + let ty = att.get("type").and_then(Value::as_str); + let mode = att.get("commandMode").and_then(Value::as_str); + if ty == Some("queued_command") && mode == Some("task-notification") { + return true; + } + } + false +} + +// --------------------------------------------------------------------------- +// Slash-command triad detector — see #438. +// +// Claude Code's slash commands (`/review`, `/init`, custom skills) emit a +// deterministic three-row sequence in the JSONL transcript: +// +// caveat — synthetic user row introducing the command. Body opens +// with the literal `Caveat:` prefix and the row is the +// apparent root of the parent chain for the next two +// rows. Carries no real user intent — it's a harness +// artifact. +// invocation — user row whose `parentUuid == caveat.uuid`. Body +// carries the synthetic command envelope: +// ``, ``, optionally +// ``. +// stdout — user row whose `parentUuid == invocation.uuid`. Body +// carries the captured stdout in a `` +// block. +// +// The classifier historically treated each row as a separate activity, +// which trebled the apparent activity count for sessions that lean on +// slash commands. The detector here collapses the triad into one +// synthetic `Skill` activity for downstream rollups; token attribution +// stays on the underlying rows so `burn hotspots` isn't double-charged. +// +// Pinning detection on parent-UUID chain shape (not on the exact text +// prefix of the caveat row) is deliberate: the literal `Caveat:` opener +// has drifted across Claude Code versions, but the chain shape — three +// rows linked caveat → invocation → stdout — has stayed stable. We use +// the text markers `` and `` as +// purpose checks on the invocation and stdout rows (so an unrelated +// three-row chain that happens to look structurally similar — e.g. a +// real user prompt followed by an assistant reply followed by a tool +// result — does NOT misdetect), but the chain-shape predicate carries +// the primary signal. See `is_task_notification` (#442) for the same +// shape-AND-purpose pattern. + +const CAVEAT_PREFIX: &str = "Caveat:"; +const COMMAND_NAME_OPEN: &str = ""; +const COMMAND_NAME_CLOSE: &str = ""; +const LOCAL_STDOUT_OPEN: &str = ""; + +/// One detected slash-command triad: the three row indices into the +/// original `rows` slice and the extracted skill name (when the +/// invocation body exposed a `` block; `None` otherwise). +/// +/// Indices are stable references into the caller's input — the +/// downstream wiring in the Claude reader uses them to override per-row +/// activity attribution without re-walking the input. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SlashTriad { + pub caveat_idx: usize, + pub invocation_idx: usize, + pub stdout_idx: usize, + pub skill_name: Option, +} + +/// Detect Claude slash-command triads in a flat slice of raw JSONL rows +/// (already-parsed as `serde_json::Map`s). +/// +/// The walk is O(n): build a `uuid -> index` index once, then for each +/// candidate caveat row look up the two children by their UUID chain. +/// Each row is consumed by at most one triad — the row-index sets are +/// disjoint by construction (we mark each used index in a bitset). +/// +/// Caller obligations: +/// +/// - The slice must preserve JSONL emission order. The detector does +/// not sort or filter — it inspects rows in place. +/// - Rows are matched on the parent-UUID chain shape; text checks on +/// the invocation and stdout rows are purpose guards that block +/// structurally-similar but semantically-different chains (e.g. a +/// normal user → assistant → tool_result chain) from misdetecting. +pub fn detect_slash_triads(rows: &[Map]) -> Vec { + if rows.len() < 3 { + return Vec::new(); + } + // Track rows already consumed by a prior triad so a single row can't + // be the invocation of triad A and the caveat of triad B simultaneously. + let mut consumed = vec![false; rows.len()]; + let mut out: Vec = Vec::new(); + + for (caveat_idx, caveat) in rows.iter().enumerate() { + if consumed[caveat_idx] { + continue; + } + if !is_caveat_row(caveat) { + continue; + } + let caveat_uuid = match caveat.get("uuid").and_then(Value::as_str) { + Some(u) if !u.is_empty() => u, + _ => continue, + }; + // The invocation row must (a) point at the caveat via parentUuid + // AND (b) carry an invocation body. The latter is the purpose + // check that blocks an unrelated child of the caveat from + // promoting the chain into a Skill. + let (invocation_idx, invocation) = match find_first_unconsumed_child( + rows, + &consumed, + caveat_uuid, + caveat_idx + 1, + is_invocation_row, + ) { + Some(pair) => pair, + None => continue, + }; + let invocation_uuid = match invocation.get("uuid").and_then(Value::as_str) { + Some(u) if !u.is_empty() => u, + _ => continue, + }; + let (stdout_idx, _stdout) = match find_first_unconsumed_child( + rows, + &consumed, + invocation_uuid, + invocation_idx + 1, + is_stdout_row, + ) { + Some(pair) => pair, + None => continue, + }; + let skill_name = extract_skill_name(invocation).or_else(|| extract_skill_name(caveat)); + consumed[caveat_idx] = true; + consumed[invocation_idx] = true; + consumed[stdout_idx] = true; + out.push(SlashTriad { + caveat_idx, + invocation_idx, + stdout_idx, + skill_name, + }); + } + out +} + +/// True when this row matches the caveat shape: a user-typed row whose +/// extracted text body begins with the `Caveat:` literal. We deliberately +/// keep this lightweight — the structural check (caveat is the root of +/// the chain a downstream invocation/stdout points at) carries the +/// primary signal. +fn is_caveat_row(row: &Map) -> bool { + if row.get("type").and_then(Value::as_str) != Some("user") { + return false; + } + extract_row_text(row) + .map(|s| s.trim_start().starts_with(CAVEAT_PREFIX)) + .unwrap_or(false) +} + +/// True when this row carries a `` block — the invocation +/// payload of a slash command. +fn is_invocation_row(row: &Map) -> bool { + if row.get("type").and_then(Value::as_str) != Some("user") { + return false; + } + extract_row_text(row) + .map(|s| s.contains(COMMAND_NAME_OPEN)) + .unwrap_or(false) +} + +/// True when this row carries a `` block — the +/// stdout-capture payload of a slash command. +fn is_stdout_row(row: &Map) -> bool { + if row.get("type").and_then(Value::as_str) != Some("user") { + return false; + } + extract_row_text(row) + .map(|s| s.contains(LOCAL_STDOUT_OPEN)) + .unwrap_or(false) +} + +/// Walk forward from `start_idx` looking for the first not-yet-consumed +/// row whose `parentUuid == parent_uuid` and which passes the purpose +/// `check`. Returns `(idx, &row)` or `None`. Forward-only because the +/// JSONL ordering for these rows is stable in practice (the harness +/// writes caveat → invocation → stdout adjacently); a sibling reorder +/// would land both the invocation and stdout *after* the caveat. +fn find_first_unconsumed_child<'a, F>( + rows: &'a [Map], + consumed: &[bool], + parent_uuid: &str, + start_idx: usize, + check: F, +) -> Option<(usize, &'a Map)> +where + F: Fn(&Map) -> bool, +{ + for (offset, row) in rows[start_idx..].iter().enumerate() { + let idx = start_idx + offset; + if consumed[idx] { + continue; + } + let pu = row.get("parentUuid").and_then(Value::as_str).unwrap_or(""); + if pu != parent_uuid { + continue; + } + if check(row) { + return Some((idx, row)); + } + } + None +} + +/// Pull `...` text from either the caveat +/// or invocation row. Returns `None` if no marker block is present. +fn extract_skill_name(row: &Map) -> Option { + let text = extract_row_text(row)?; + let open = text.find(COMMAND_NAME_OPEN)?; + let after = &text[open + COMMAND_NAME_OPEN.len()..]; + let close = after.find(COMMAND_NAME_CLOSE)?; + let raw = after[..close].trim(); + if raw.is_empty() { + return None; + } + // Tolerate the optional leading `/` (matches the ghost_surface + // miner's accept-with-or-without convention). + Some(raw.trim_start_matches('/').to_string()) +} + +/// Extract the row's plain text body (string content, or concatenation +/// of `text` / `content` strings inside an array body). Mirrors +/// `extract_plain_user_text_from_obj` from the Claude reader closely +/// enough to detect markers; we read the `content` field of a user-typed +/// row whose body may be a string or a list of blocks. +fn extract_row_text(row: &Map) -> Option { + let body = row.get("message").and_then(|m| m.get("content"))?; + if let Some(s) = body.as_str() { + if s.is_empty() { + return None; + } + return Some(s.to_string()); + } + let arr = body.as_array()?; + let mut parts: Vec = Vec::new(); + for block in arr { + let bo = match block.as_object() { + Some(o) => o, + None => continue, + }; + // `{"type":"text","text":"..."}` — assistant-style text block. + if bo.get("type").and_then(Value::as_str) == Some("text") { + if let Some(s) = bo.get("text").and_then(Value::as_str) { + if !s.is_empty() { + parts.push(s.to_string()); + } + } + continue; + } + // `{"type":"tool_result","content":"..."}` — string-content tool + // result envelope. The slash-command stdout row can ship its + // payload this way; we still want the marker visible to the + // purpose check. + if let Some(s) = bo.get("content").and_then(Value::as_str) { + if !s.is_empty() { + parts.push(s.to_string()); + } + } + } + if parts.is_empty() { + None + } else { + Some(parts.join("\n")) + } +} diff --git a/crates/relayburn-sdk/src/reader/claude.rs b/crates/relayburn-sdk/src/reader/claude.rs index e1f974f2..ac984035 100644 --- a/crates/relayburn-sdk/src/reader/claude.rs +++ b/crates/relayburn-sdk/src/reader/claude.rs @@ -41,27 +41,28 @@ mod parent_chain; use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fs::File; -use std::io::{BufRead, BufReader, Read, Seek, SeekFrom}; use std::path::Path; use serde_json::Value; use self::parent_chain::ChainNode; -use crate::reader::classifier::{ - classify_activity, detect_slash_triads, is_task_notification, ClassificationInput, -}; -use crate::reader::git::resolve_project; +use crate::reader::classifier::{classify_activity, ClassificationInput}; use crate::reader::hash::{args_hash, content_hash}; -use crate::reader::inference::{RequestIdLookup, TurnKey}; +use crate::reader::inference::RequestIdLookup; use crate::reader::types::{ ActivityCategory, CompactionEvent, ContentKind, ContentRecord, ContentRole, ContentStoreMode, - ContentToolResult, ContentToolUse, Coverage, Fidelity, RelationshipSourceKind, - RelationshipType, SessionRelationshipRecord, SourceKind, StopReason, Subagent, ToolCall, - ToolResultEventRecord, TurnRecord, Usage, UsageGranularity, UserTurnBlock, UserTurnRecord, + ContentToolResult, ContentToolUse, Coverage, Fidelity, SessionRelationshipRecord, SourceKind, + Subagent, ToolCall, ToolResultEventRecord, TurnRecord, Usage, UsageGranularity, UserTurnBlock, + UserTurnRecord, }; use crate::reader::user_turn::{HeuristicCounter, TokenCounter}; +// Re-exported into the conformance test module via its `use super::*;`. The +// production parse engine that referenced these directly now lives in the +// `incremental` submodule, so the root only needs them under `cfg(test)`. +#[cfg(test)] +use crate::reader::types::{RelationshipSourceKind, RelationshipType, StopReason}; + // Discovery + pairing for Task subagent sidecar transcripts. Public so the // SDK surface (`crate::reader::{discover_subagents, pair_to_main, // SubagentTranscript}`) and the ingest path can both reach it. Lazy — @@ -80,13 +81,6 @@ pub mod span_tree; // parse engine below drives these helpers. mod relationships; -use self::relationships::{ - annotate_compaction_events, annotate_relationships_with_evidence, annotate_spawn_events, - collect_explicit_claude_relationships_incremental, collect_subagent_relationships, - derive_file_session_id_from_parts, emit_local_continuation_from_resume, new_evidence, - record_evidence_from_line, record_explicit_relationship_evidence, record_resume_marker, - RelationshipKey, -}; pub use self::relationships::{ reconcile_claude_session_relationships, ClaudeRelationshipEvidence, ReconcileClaudeRelationshipsInput, @@ -97,10 +91,16 @@ pub use self::relationships::{ // this file; the parse engine below drives these helpers per line. mod tool_results; -use self::tool_results::{ - build_claude_system_tool_result_event, collect_replacement_meta, collect_tool_result_events, - ReplacementMeta, -}; +use self::tool_results::ReplacementMeta; + +// Incremental parse engine: the resume prescan, the `ClaudeParseState` +// streaming state machine, and the `run_incremental` driver the public +// `parse_claude_session*` entry points wrap. Split out of this file; the +// helpers/types above feed it and it drives the relationships/tool_results +// helpers per line. +mod incremental; + +use self::incremental::run_incremental; // --------------------------------------------------------------------------- // Public surface. @@ -246,7 +246,7 @@ pub fn parse_claude_session_incremental_with_counter, C: TokenCou // --------------------------------------------------------------------------- #[derive(Debug, Default, Clone, Copy)] -struct UsageCoverage { +pub(in crate::reader::claude) struct UsageCoverage { has_input_tokens: bool, has_output_tokens: bool, has_cache_read_tokens: bool, @@ -254,17 +254,17 @@ struct UsageCoverage { } #[derive(Debug, Clone)] -struct WorkingRecord { - message_id: String, - first_ts: String, - model: String, - session_id: String, - cwd: Option, +pub(in crate::reader::claude) struct WorkingRecord { + pub(in crate::reader::claude) message_id: String, + pub(in crate::reader::claude) first_ts: String, + pub(in crate::reader::claude) model: String, + pub(in crate::reader::claude) session_id: String, + pub(in crate::reader::claude) cwd: Option, is_sidechain: bool, - usage: Usage, - usage_coverage: UsageCoverage, - blocks: Vec, - stop_reason: Option, + pub(in crate::reader::claude) usage: Usage, + pub(in crate::reader::claude) usage_coverage: UsageCoverage, + pub(in crate::reader::claude) blocks: Vec, + pub(in crate::reader::claude) stop_reason: Option, first_assistant_uuid: Option, #[allow(dead_code)] parent_assistant_uuid: Option, @@ -273,7 +273,7 @@ struct WorkingRecord { /// per-API-call key (see `reader/inference.rs` and issue #434). /// `None` when no row in this group emitted one — the inference /// builder falls back to `message_id`. - request_id: Option, + pub(in crate::reader::claude) request_id: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -290,7 +290,7 @@ struct AgentToolUse { } #[derive(Debug, Clone)] -struct LineNode { +pub(in crate::reader::claude) struct LineNode { uuid: String, parent_uuid: Option, kind: LineKind, @@ -355,7 +355,7 @@ fn nearest_user_prompt_root(start_uuid: &str, nodes: &HashMap) } #[derive(Debug, Clone)] -struct InvocationInfo { +pub(in crate::reader::claude) struct InvocationInfo { root_uuid: String, parent_tool_use_id: Option, subagent_type: Option, @@ -367,7 +367,7 @@ struct InvocationInfo { // Line ingest helpers. // --------------------------------------------------------------------------- -fn ingest_assistant_record( +pub(in crate::reader::claude) fn ingest_assistant_record( parsed: &Value, obj: &serde_json::Map, working: &mut HashMap, @@ -493,7 +493,10 @@ fn make_line_node(line: &Value, kind: LineKind) -> Option { }) } -fn register_assistant_node(line: &Value, nodes: &mut HashMap) { +pub(in crate::reader::claude) fn register_assistant_node( + line: &Value, + nodes: &mut HashMap, +) { let mut node = match make_line_node(line, LineKind::Assistant) { Some(n) => n, None => return, @@ -539,7 +542,11 @@ fn register_assistant_node(line: &Value, nodes: &mut HashMap) nodes.insert(node.uuid.clone(), node); } -fn register_user_node(line: &Value, nodes: &mut HashMap, is_user_prompt: bool) { +pub(in crate::reader::claude) fn register_user_node( + line: &Value, + nodes: &mut HashMap, + is_user_prompt: bool, +) { let mut node = match make_line_node(line, LineKind::User) { Some(n) => n, None => return, @@ -644,7 +651,7 @@ fn merge_usage_coverage(a: &UsageCoverage, b: &UsageCoverage) -> UsageCoverage { } } -fn build_claude_fidelity(uc: &UsageCoverage) -> Fidelity { +pub(in crate::reader::claude) fn build_claude_fidelity(uc: &UsageCoverage) -> Fidelity { let coverage = Coverage { has_input_tokens: uc.has_input_tokens, has_output_tokens: uc.has_output_tokens, @@ -663,7 +670,7 @@ fn build_claude_fidelity(uc: &UsageCoverage) -> Fidelity { // Tool calls / files-touched. // --------------------------------------------------------------------------- -fn extract_tool_calls( +pub(in crate::reader::claude) fn extract_tool_calls( blocks: &[Value], errored: &HashSet, replacement: Option<&HashMap>, @@ -761,7 +768,7 @@ fn pick_target(name: &str, input: &Value) -> Option { } } -fn extract_files_touched(tool_calls: &[ToolCall]) -> Vec { +pub(in crate::reader::claude) fn extract_files_touched(tool_calls: &[ToolCall]) -> Vec { let mut seen = HashSet::new(); let mut out = Vec::new(); for tc in tool_calls { @@ -782,7 +789,7 @@ fn extract_files_touched(tool_calls: &[ToolCall]) -> Vec { // Subagent invocation resolution. // --------------------------------------------------------------------------- -fn resolve_subagent( +pub(in crate::reader::claude) fn resolve_subagent( w: &WorkingRecord, nodes: &HashMap, cache: &mut HashMap>, @@ -887,7 +894,9 @@ fn resolve_invocation( // Content extraction. // --------------------------------------------------------------------------- -fn extract_assistant_content(w: &WorkingRecord) -> Vec { +pub(in crate::reader::claude) fn extract_assistant_content( + w: &WorkingRecord, +) -> Vec { let mut out = Vec::new(); if w.session_id.is_empty() || w.message_id.is_empty() { return out; @@ -971,7 +980,9 @@ fn extract_assistant_content(w: &WorkingRecord) -> Vec { out } -fn extract_user_content(line: &serde_json::Map) -> Vec { +pub(in crate::reader::claude) fn extract_user_content( + line: &serde_json::Map, +) -> Vec { let mut out = Vec::new(); let session_id = string_field(line, SESSION_ID_KEYS, false).unwrap_or_default(); let message_id = string_field(line, &["uuid"], false).unwrap_or_default(); @@ -1068,7 +1079,7 @@ fn extract_user_content(line: &serde_json::Map) -> Vec( +pub(in crate::reader::claude) fn build_user_turn_record( line: &serde_json::Map, preceding_message_id: Option<&str>, counter: &C, @@ -1180,7 +1191,10 @@ pub(super) fn extract_plain_user_text_from_obj( } } -fn collect_errored_tool_use_ids(line: &serde_json::Map, into: &mut HashSet) { +pub(in crate::reader::claude) fn collect_errored_tool_use_ids( + line: &serde_json::Map, + into: &mut HashSet, +) { let arr = match line .get("message") .and_then(|m| m.get("content")) @@ -1206,7 +1220,7 @@ fn collect_errored_tool_use_ids(line: &serde_json::Map, into: &mu } } -fn apply_classification( +pub(in crate::reader::claude) fn apply_classification( record: &mut TurnRecord, w: &WorkingRecord, user_text_by_message_id: &HashMap, @@ -1289,692 +1303,6 @@ fn extract_assistant_text_for_classification(blocks: &[Value]) -> String { parts.join("\n") } -// --------------------------------------------------------------------------- -// Incremental parser. -// --------------------------------------------------------------------------- - -struct PrescanOutput { - last_assistant_message_id: Option, - next_event_index: u64, -} - -/// Pre-read the already-ingested prefix `[0, end_offset)` so a resumed call -/// has the same node graph, evidence, tool-result counters, event index, and -/// last-assistant-message-id it would have if it had started from byte 0. -/// Mirrors `prescanNodes` in `packages/reader/src/claude.ts`. -fn prescan_nodes( - path: &Path, - end_offset: u64, - nodes_by_uuid: &mut HashMap, - evidence: &mut ClaudeRelationshipEvidence, - tool_result_counters: &mut HashMap, -) -> std::io::Result { - if end_offset == 0 { - return Ok(PrescanOutput { - last_assistant_message_id: None, - next_event_index: 0, - }); - } - let file = File::open(path)?; - let size = file.metadata()?.len(); - let length = end_offset.min(size); - if length == 0 { - return Ok(PrescanOutput { - last_assistant_message_id: None, - next_event_index: 0, - }); - } - // Stream the prefix line-by-line rather than reading `[0, length)` - // into memory all at once. For multi-GB sessions the up-front - // `vec![0u8; length as usize]` was a multi-GB allocation we never - // need — only the longest single line has to fit in memory. - let mut reader = BufReader::new(file).take(length); - let mut line_buf: Vec = Vec::new(); - let mut last_assistant_message_id: Option = None; - let mut next_event_index: u64 = 0; - loop { - line_buf.clear(); - let n = reader.read_until(b'\n', &mut line_buf)?; - if n == 0 { - break; - } - // A trailing partial line (no `\n`) inside the prescan window - // should never happen — incremental ingest only commits cursors - // at newline boundaries — but guard anyway. - if line_buf.last() != Some(&b'\n') { - break; - } - let raw = std::str::from_utf8(&line_buf[..n - 1]).unwrap_or("").trim(); - if raw.is_empty() { - continue; - } - let parsed: Value = match serde_json::from_str(raw) { - Ok(v) => v, - Err(_) => continue, - }; - let obj = match parsed.as_object() { - Some(o) => o.clone(), - None => continue, - }; - let line_type = obj.get("type").and_then(Value::as_str).unwrap_or(""); - match line_type { - "assistant" => { - register_assistant_node(&parsed, nodes_by_uuid); - record_evidence_from_line(evidence, &parsed); - record_explicit_relationship_evidence(evidence, &obj); - if let Some(mid) = obj - .get("message") - .and_then(|m| m.get("id")) - .and_then(Value::as_str) - { - last_assistant_message_id = Some(mid.to_string()); - } - } - "user" => { - // Match the main-loop classification: a row is a real - // user-prompt root only when it isn't a harness task - // notification AND carries plain user text. See #439 for - // the task-notification gate and #433 for why we tag - // the LineNode for the parent-chain walker. - let is_user_prompt = !is_task_notification(&obj) - && extract_plain_user_text_from_obj(&obj).is_some_and(|s| !s.is_empty()); - register_user_node(&parsed, nodes_by_uuid, is_user_prompt); - record_evidence_from_line(evidence, &parsed); - record_explicit_relationship_evidence(evidence, &obj); - record_resume_marker(evidence, &obj); - let mut harvested: Vec = Vec::new(); - next_event_index = collect_tool_result_events( - &obj, - &mut harvested, - tool_result_counters, - next_event_index, - ); - } - "system" - if build_claude_system_tool_result_event( - &obj, - tool_result_counters, - next_event_index, - ) - .is_some() => - { - next_event_index += 1; - } - _ => {} - } - } - Ok(PrescanOutput { - last_assistant_message_id, - next_event_index, - }) -} - -fn record_root_incremental( - out: &mut Vec<(u64, SessionRelationshipRecord)>, - seen: &mut HashSet, - session_id: &str, - ts: Option<&str>, - line_offset: u64, - file_session_id: Option<&str>, -) { - let canonical = file_session_id.unwrap_or(session_id).to_string(); - if !seen.insert(canonical.clone()) { - return; - } - let mut row = SessionRelationshipRecord { - v: 1, - source: RelationshipSourceKind::ClaudeCode, - session_id: canonical, - related_session_id: None, - relationship_type: RelationshipType::Root, - ts: None, - source_session_id: None, - source_version: None, - parent_tool_use_id: None, - agent_id: None, - subagent_type: None, - description: None, - }; - if let Some(t) = ts { - if !t.is_empty() { - row.ts = Some(t.to_string()); - } - } - out.push((line_offset, row)); -} - -fn run_incremental( - path: &Path, - options: &ParseIncrementalOptions, - counter: &C, - emit_in_progress: bool, -) -> std::io::Result { - let start_offset = options.start_offset.unwrap_or(0); - let content_mode = options.content_mode.unwrap_or(ContentStoreMode::Off); - let capture_content = matches!(content_mode, ContentStoreMode::Full); - - let file_session_id = derive_file_session_id_from_parts( - options.file_session_id.as_deref(), - options.session_path.as_deref(), - ); - let mut evidence = new_evidence(file_session_id.clone()); - - let metadata = std::fs::metadata(path)?; - let size = metadata.len(); - if start_offset >= size { - return Ok(ParseIncrementalResult { - turns: Vec::new(), - content: Vec::new(), - events: Vec::new(), - relationships: Vec::new(), - tool_result_events: Vec::new(), - user_turns: Vec::new(), - request_id_lookup: RequestIdLookup::new(), - end_offset: start_offset, - last_user_text: options.last_user_text.clone().unwrap_or_default(), - evidence, - }); - } - - let mut nodes_by_uuid: HashMap = HashMap::new(); - let mut invocation_cache: HashMap> = HashMap::new(); - let mut tool_result_counters: HashMap = HashMap::new(); - let mut next_event_index: u64 = 0; - let mut last_assistant_message_id: Option = None; - - if start_offset > 0 { - let pre = prescan_nodes( - path, - start_offset, - &mut nodes_by_uuid, - &mut evidence, - &mut tool_result_counters, - )?; - last_assistant_message_id = pre.last_assistant_message_id; - next_event_index = pre.next_event_index; - } - - // -1 sentinel: resume marker came from the prescan (definitely emit). - // u64::MAX sentinel: no resume marker yet seen. - // Otherwise: byte offset of the line that first set the marker on this pass. - let mut resume_marker_offset: u64 = if evidence.has_resume_marker { - 0 - } else { - u64::MAX - }; - - let mut current_user_text = options.last_user_text.clone().unwrap_or_default(); - - let mut working: HashMap = HashMap::new(); - let mut order: Vec = Vec::new(); - let mut message_id_first_offset: HashMap = HashMap::new(); - // Legacy file-order map kept as a fallback when the assistant row - // lacks a `uuid` or its parent chain doesn't terminate at a known - // user prompt. The preferred lookup is `user_text_by_uuid` walked - // via `nearest_user_prompt_root` (#433). - let mut user_text_by_message_id: HashMap = HashMap::new(); - // User-prompt text keyed by the user line's own `uuid` — read by the - // parent-chain walker during turn classification. Populated only for - // real user prompts (task notifications excluded; empty bodies - // excluded). - let mut user_text_by_uuid: HashMap = HashMap::new(); - let mut errored_tool_use_ids: HashSet = HashSet::new(); - let mut replacement_meta_by_tool_use_id: HashMap = HashMap::new(); - // Slash-command triad detection (#438) needs a flat slice of user-typed - // rows to look up the parent-UUID chain shape. We accumulate only the - // minimal field set the detector reads (`type`, `uuid`, `parentUuid`, - // `message.content`) so memory stays bounded — three rows per triad, - // and only user-typed rows are stored. Detection runs once after the - // streaming loop closes; the resulting `skill_uuids` set is consulted - // by `apply_classification` to override the activity to `Skill`. - let mut user_rows_for_triad: Vec> = Vec::new(); - let mut events: Vec<(u64, CompactionEvent)> = Vec::new(); - let mut pending_user_content: Vec<(u64, ContentRecord)> = Vec::new(); - let mut pending_tool_result_events: Vec<(u64, ToolResultEventRecord)> = Vec::new(); - let mut pending_relationships: Vec<(u64, SessionRelationshipRecord)> = Vec::new(); - let mut pending_user_turns: Vec<(u64, UserTurnRecord)> = Vec::new(); - let mut seen_root_session_ids: HashSet = HashSet::new(); - let mut seen_explicit_relationship_ids: HashSet = HashSet::new(); - let mut pending_user_turn_inc_idx: Option = None; - - let mut file = File::open(path)?; - file.seek(SeekFrom::Start(start_offset))?; - // Stream from `start_offset` line-by-line. The previous implementation - // allocated `Vec::with_capacity((size - start_offset) as usize)` and - // `read_to_end` into it — for a multi-GB session this was a multi-GB - // up-front allocation. With BufReader + `read_until` only the longest - // single line stays resident. - let mut reader = BufReader::new(file); - let mut line_buf: Vec = Vec::new(); - let mut cursor_offset: u64 = start_offset; // position past last complete \n - loop { - line_buf.clear(); - let n = reader.read_until(b'\n', &mut line_buf)?; - if n == 0 { - break; - } - // Drop trailing partial lines — the next incremental call resumes - // from `cursor_offset`, which we only advance past complete `\n`. - // `emit_in_progress` runs from the single-shot full-parse entry where - // there is no next call, so a final line without a trailing `\n` - // (truncated write, unflushed `\n`) must still be processed; the old - // `ParseState` path used `read_line` and surfaced it. - let has_newline = line_buf.last() == Some(&b'\n'); - if !has_newline && !emit_in_progress { - break; - } - let line_start_offset = cursor_offset; - let line_end_offset = cursor_offset + n as u64; - let body_end = if has_newline { n - 1 } else { n }; - let trimmed = std::str::from_utf8(&line_buf[..body_end]) - .unwrap_or("") - .trim(); - // Single-shot callers have no next pass, so a final partial line still - // needs to bump the cursor past its body — `end_offset = cursor_offset` - // is what the per-record offset filters compare against below. - if has_newline || emit_in_progress { - cursor_offset = line_end_offset; - } - if trimmed.is_empty() { - continue; - } - let parsed: Value = match serde_json::from_str(trimmed) { - Ok(v) => v, - Err(_) => continue, - }; - let obj = match parsed.as_object() { - Some(o) => o.clone(), - None => continue, - }; - let line_type = obj.get("type").and_then(Value::as_str).unwrap_or(""); - match line_type { - "assistant" => { - let mid = obj - .get("message") - .and_then(|m| m.get("id")) - .and_then(Value::as_str) - .map(str::to_string); - if let Some(ref mid_str) = mid { - if let Some(idx) = pending_user_turn_inc_idx { - if !message_id_first_offset.contains_key(mid_str) { - pending_user_turns[idx].1.following_message_id = Some(mid_str.clone()); - pending_user_turn_inc_idx = None; - } - } - message_id_first_offset - .entry(mid_str.clone()) - .or_insert(line_start_offset); - user_text_by_message_id - .entry(mid_str.clone()) - .or_insert_with(|| current_user_text.clone()); - last_assistant_message_id = Some(mid_str.clone()); - } - let session_id = string_field(&obj, SESSION_ID_KEYS, false); - let timestamp = string_field(&obj, TIMESTAMP_KEYS, false); - if let Some(ref sid) = session_id { - if !sid.is_empty() { - record_root_incremental( - &mut pending_relationships, - &mut seen_root_session_ids, - sid, - timestamp.as_deref(), - line_start_offset, - file_session_id.as_deref(), - ); - collect_explicit_claude_relationships_incremental( - &obj, - &mut evidence, - &mut pending_relationships, - &mut seen_explicit_relationship_ids, - file_session_id.as_deref().unwrap_or(sid.as_str()), - timestamp.as_deref(), - line_start_offset, - ); - } - } - record_evidence_from_line(&mut evidence, &parsed); - ingest_assistant_record( - &parsed, - &obj, - &mut working, - &mut order, - &mut nodes_by_uuid, - ); - } - "user" => { - // Slash-command triad detector (#438) keeps a slim copy of - // every user-typed row so a post-loop pass can find the - // caveat → invocation → stdout chain shape. We clone the - // row before the rest of the branch consumes it; the - // detector only reads four fields so memory stays modest - // (one entry per user row, dropped at function exit). - user_rows_for_triad.push(obj.clone()); - // Harness-injected `` rows share the user - // envelope but represent system events, not real prompts. - // Detecting them here keeps them out of `current_user_text` - // (so the classifier doesn't get task-notification text as - // "user intent") and out of `pending_user_turns` (so - // user-turn aggregates aren't inflated). Side effects like - // session-relationship discovery still run because those - // are independent of "is this a real user prompt". See - // AgentWorkforce/burn#439. - let task_notification = is_task_notification(&obj); - let user_text = if task_notification { - None - } else { - extract_plain_user_text_from_obj(&obj).filter(|s| !s.is_empty()) - }; - let is_user_prompt = user_text.is_some(); - register_user_node(&parsed, &mut nodes_by_uuid, is_user_prompt); - if let Some(ref text) = user_text { - current_user_text = text.clone(); - // Index by the user line's UUID for the parent-chain - // walker (#433). Falls back to no-op when the row - // lacks a `uuid`, in which case file-order remains - // the only association mechanism for downstream - // assistants. - if let Some(uuid) = obj.get("uuid").and_then(Value::as_str) { - if !uuid.is_empty() { - user_text_by_uuid - .entry(uuid.to_string()) - .or_insert_with(|| text.clone()); - } - } - } - collect_errored_tool_use_ids(&obj, &mut errored_tool_use_ids); - collect_replacement_meta(&obj, &mut replacement_meta_by_tool_use_id); - let session_id = string_field(&obj, SESSION_ID_KEYS, false); - let timestamp = string_field(&obj, TIMESTAMP_KEYS, false); - if let Some(ref sid) = session_id { - if !sid.is_empty() { - record_root_incremental( - &mut pending_relationships, - &mut seen_root_session_ids, - sid, - timestamp.as_deref(), - line_start_offset, - file_session_id.as_deref(), - ); - collect_explicit_claude_relationships_incremental( - &obj, - &mut evidence, - &mut pending_relationships, - &mut seen_explicit_relationship_ids, - file_session_id.as_deref().unwrap_or(sid.as_str()), - timestamp.as_deref(), - line_start_offset, - ); - } - } - record_evidence_from_line(&mut evidence, &parsed); - let had_resume_before = evidence.has_resume_marker; - record_resume_marker(&mut evidence, &obj); - if !had_resume_before && evidence.has_resume_marker { - resume_marker_offset = line_start_offset; - } - let mut harvested: Vec = Vec::new(); - next_event_index = collect_tool_result_events( - &obj, - &mut harvested, - &mut tool_result_counters, - next_event_index, - ); - for ev in harvested { - pending_tool_result_events.push((line_start_offset, ev)); - } - if !task_notification { - if let Some(record) = - build_user_turn_record(&obj, last_assistant_message_id.as_deref(), counter) - { - let idx = pending_user_turns.len(); - pending_user_turns.push((line_start_offset, record)); - pending_user_turn_inc_idx = Some(idx); - } - } - if capture_content { - for c in extract_user_content(&obj) { - pending_user_content.push((line_start_offset, c)); - } - } - } - "system" => { - if obj.get("subtype").and_then(Value::as_str) == Some("compact_boundary") { - let session_id = string_field(&obj, SESSION_ID_KEYS, false).unwrap_or_default(); - let ts = string_field(&obj, TIMESTAMP_KEYS, false).unwrap_or_default(); - if !session_id.is_empty() { - let mut ev = CompactionEvent { - v: 1, - source: SourceKind::ClaudeCode, - session_id, - ts, - preceding_message_id: None, - tokens_before_compact: None, - }; - if let Some(ref last) = last_assistant_message_id { - ev.preceding_message_id = Some(last.clone()); - } - events.push((line_start_offset, ev)); - } - } - if let Some(ev) = build_claude_system_tool_result_event( - &obj, - &mut tool_result_counters, - next_event_index, - ) { - pending_tool_result_events.push((line_start_offset, ev)); - next_event_index += 1; - } - } - _ => {} - } - } - - // end_offset = byte position of the earliest in-progress messageId, or - // cursor_offset (= position past the last complete newline) when all - // messages are complete. In `emit_in_progress` mode (the full - // non-incremental parse) we keep cursor_offset and emit every turn so the - // result matches the single-shot ParseResult contract: callers want every - // record we saw, including trailing in-progress assistants. - let end_offset = if emit_in_progress { - cursor_offset - } else { - let mut earliest_incomplete: Option = None; - for id in &order { - let w = match working.get(id) { - Some(w) => w, - None => continue, - }; - if w.stop_reason.is_none() { - if let Some(off) = message_id_first_offset.get(id) { - if earliest_incomplete.is_none_or(|e| *off < e) { - earliest_incomplete = Some(*off); - } - } - } - } - earliest_incomplete.unwrap_or(cursor_offset) - }; - - // Slash-command triad detection (#438). Run once over the accumulated - // user rows; the resulting set of triad UUIDs (caveat + invocation + - // stdout) is consulted by `apply_classification` to override the - // assistant turn's activity to `Skill` when its parent-chain root - // lands on any one of the three triad rows. Token attribution stays - // on the underlying turn's `usage` — the synthetic `Skill` label is - // a view, not a billing reattribution. - let mut skill_uuids: HashSet = HashSet::new(); - for triad in detect_slash_triads(&user_rows_for_triad) { - for idx in [triad.caveat_idx, triad.invocation_idx, triad.stdout_idx] { - if let Some(uuid) = user_rows_for_triad - .get(idx) - .and_then(|r| r.get("uuid")) - .and_then(Value::as_str) - { - if !uuid.is_empty() { - skill_uuids.insert(uuid.to_string()); - } - } - } - } - // Detector input is no longer needed; drop the cloned rows so we - // don't carry them past the emission loop. - drop(user_rows_for_triad); - - // Emit completed turns. In-progress messages (no stop_reason) are deferred - // — `end_offset` already backs up to before their first byte so the next - // call re-reads them. `emit_in_progress` opts the non-incremental path out - // of that skip so it emits every working record. - let mut turns: Vec = Vec::new(); - let mut assistant_pending: Vec<(u64, usize, ContentRecord)> = Vec::new(); - for (i, id) in order.iter().enumerate() { - let w = match working.get(id) { - Some(w) => w, - None => continue, - }; - if !emit_in_progress && w.stop_reason.is_none() { - continue; - } - let tool_calls = extract_tool_calls( - &w.blocks, - &errored_tool_use_ids, - Some(&replacement_meta_by_tool_use_id), - ); - let files_touched = extract_files_touched(&tool_calls); - let subagent = resolve_subagent(w, &nodes_by_uuid, &mut invocation_cache); - - let mut record = TurnRecord { - v: 1, - source: SourceKind::ClaudeCode, - session_id: w.session_id.clone(), - session_path: options.session_path.clone(), - message_id: w.message_id.clone(), - turn_index: i as u64, - ts: w.first_ts.clone(), - model: w.model.clone(), - project: None, - project_key: None, - usage: w.usage.clone(), - tool_calls: tool_calls.clone(), - files_touched: if files_touched.is_empty() { - None - } else { - Some(files_touched) - }, - subagent, - stop_reason: w - .stop_reason - .as_deref() - .map(|s| StopReason::from_wire(s).unwrap_or(StopReason::Silent)), - activity: None, - retries: None, - has_edits: None, - fidelity: Some(build_claude_fidelity(&w.usage_coverage)), - }; - if let Some(ref cwd) = w.cwd { - let resolved = resolve_project(cwd); - record.project = Some(resolved.project); - record.project_key = resolved.project_key; - } - apply_classification( - &mut record, - w, - &user_text_by_message_id, - &user_text_by_uuid, - &nodes_by_uuid, - &errored_tool_use_ids, - &skill_uuids, - ); - turns.push(record); - - if capture_content { - let msg_offset = *message_id_first_offset.get(&w.message_id).unwrap_or(&0); - for (sub, r) in extract_assistant_content(w).into_iter().enumerate() { - assistant_pending.push((msg_offset, sub + 1, r)); - } - } - } - - // Filter content by end_offset and interleave by source-line offset. - // appendContent has no row-level dedup, so we MUST drop rows past - // end_offset — the next call will re-read those bytes and re-emit them. - let mut content: Vec = Vec::new(); - if capture_content { - let mut merged: Vec<(u64, usize, ContentRecord)> = Vec::new(); - for (off, rec) in pending_user_content.into_iter() { - if off < end_offset { - merged.push((off, 0, rec)); - } - } - for (off, sub, rec) in assistant_pending.into_iter() { - if off < end_offset { - merged.push((off, sub, rec)); - } - } - merged.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); - content = merged.into_iter().map(|(_, _, r)| r).collect(); - } - - let mut emitted_events: Vec = events - .into_iter() - .filter(|(off, _)| *off < end_offset) - .map(|(_, ev)| ev) - .collect(); - annotate_compaction_events(&mut emitted_events, &turns); - - let mut emitted_relationships: Vec = pending_relationships - .into_iter() - .filter(|(off, _)| *off < end_offset) - .map(|(_, r)| r) - .collect(); - collect_subagent_relationships(&turns, &mut emitted_relationships); - if resume_marker_offset < end_offset { - emit_local_continuation_from_resume(&mut emitted_relationships, &evidence); - } - annotate_relationships_with_evidence(&mut emitted_relationships, &evidence); - - let mut emitted_tool_result_events: Vec = pending_tool_result_events - .into_iter() - .filter(|(off, _)| *off < end_offset) - .map(|(_, ev)| ev) - .collect(); - annotate_spawn_events(&mut emitted_tool_result_events, &turns); - - let emitted_user_turns: Vec = pending_user_turns - .into_iter() - .filter(|(off, _)| *off < end_offset) - .map(|(_, ut)| ut) - .collect(); - - // Build the `(source, session_id, message_id) -> requestId` lookup - // for every emitted turn. We only walk turns the run actually emits - // (in-progress assistant rows are filtered out above) so the lookup - // entries always correspond to an outbound `TurnRecord`. See issue - // #434. - let mut request_id_lookup = RequestIdLookup::new(); - for t in &turns { - if let Some(w) = working.get(&t.message_id) { - if let Some(req) = w.request_id.as_ref() { - if !req.is_empty() { - request_id_lookup.insert(TurnKey::for_turn(t), req.clone()); - } - } - } - } - - Ok(ParseIncrementalResult { - turns, - content, - events: emitted_events, - relationships: emitted_relationships, - tool_result_events: emitted_tool_result_events, - user_turns: emitted_user_turns, - request_id_lookup, - end_offset, - last_user_text: current_user_text, - evidence, - }) -} - // --------------------------------------------------------------------------- // Misc helpers. // --------------------------------------------------------------------------- diff --git a/crates/relayburn-sdk/src/reader/claude/incremental.rs b/crates/relayburn-sdk/src/reader/claude/incremental.rs new file mode 100644 index 00000000..78f82042 --- /dev/null +++ b/crates/relayburn-sdk/src/reader/claude/incremental.rs @@ -0,0 +1,844 @@ +//! Claude incremental-parse engine. +//! +//! Mechanically split out of `reader/claude.rs` to shrink the oversized root +//! file. Holds the resume prescan (`prescan_nodes`), the per-line streaming +//! state machine (`ClaudeParseState`), and the driver `run_incremental` that +//! the public `parse_claude_session*` entry points wrap. Behavior is +//! unchanged from the inline implementation; only module placement, +//! visibility, and imports differ. + +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::{BufRead, BufReader, Read, Seek, SeekFrom}; +use std::path::Path; + +use serde_json::Value; + +use crate::reader::classifier::{detect_slash_triads, is_task_notification}; +use crate::reader::git::resolve_project; +use crate::reader::inference::{RequestIdLookup, TurnKey}; +use crate::reader::types::{ + CompactionEvent, ContentRecord, ContentStoreMode, RelationshipSourceKind, RelationshipType, + SessionRelationshipRecord, SourceKind, StopReason, ToolResultEventRecord, TurnRecord, + UserTurnRecord, +}; +use crate::reader::user_turn::TokenCounter; + +use super::relationships::{ + annotate_compaction_events, annotate_relationships_with_evidence, annotate_spawn_events, + collect_explicit_claude_relationships_incremental, collect_subagent_relationships, + derive_file_session_id_from_parts, emit_local_continuation_from_resume, new_evidence, + record_evidence_from_line, record_explicit_relationship_evidence, record_resume_marker, + ClaudeRelationshipEvidence, RelationshipKey, +}; +use super::tool_results::{ + build_claude_system_tool_result_event, collect_replacement_meta, collect_tool_result_events, + ReplacementMeta, +}; +use super::{ + apply_classification, build_claude_fidelity, build_user_turn_record, + collect_errored_tool_use_ids, extract_assistant_content, extract_files_touched, + extract_plain_user_text_from_obj, extract_tool_calls, extract_user_content, + ingest_assistant_record, register_assistant_node, register_user_node, resolve_subagent, + string_field, InvocationInfo, LineNode, ParseIncrementalOptions, ParseIncrementalResult, + WorkingRecord, SESSION_ID_KEYS, TIMESTAMP_KEYS, +}; + +struct PrescanOutput { + last_assistant_message_id: Option, + next_event_index: u64, +} + +/// Pre-read the already-ingested prefix `[0, end_offset)` so a resumed call +/// has the same node graph, evidence, tool-result counters, event index, and +/// last-assistant-message-id it would have if it had started from byte 0. +/// Mirrors `prescanNodes` in `packages/reader/src/claude.ts`. +fn prescan_nodes( + path: &Path, + end_offset: u64, + nodes_by_uuid: &mut HashMap, + evidence: &mut ClaudeRelationshipEvidence, + tool_result_counters: &mut HashMap, +) -> std::io::Result { + if end_offset == 0 { + return Ok(PrescanOutput { + last_assistant_message_id: None, + next_event_index: 0, + }); + } + let file = File::open(path)?; + let size = file.metadata()?.len(); + let length = end_offset.min(size); + if length == 0 { + return Ok(PrescanOutput { + last_assistant_message_id: None, + next_event_index: 0, + }); + } + // Stream the prefix line-by-line rather than reading `[0, length)` + // into memory all at once. For multi-GB sessions the up-front + // `vec![0u8; length as usize]` was a multi-GB allocation we never + // need — only the longest single line has to fit in memory. + let mut reader = BufReader::new(file).take(length); + let mut line_buf: Vec = Vec::new(); + let mut last_assistant_message_id: Option = None; + let mut next_event_index: u64 = 0; + loop { + line_buf.clear(); + let n = reader.read_until(b'\n', &mut line_buf)?; + if n == 0 { + break; + } + // A trailing partial line (no `\n`) inside the prescan window + // should never happen — incremental ingest only commits cursors + // at newline boundaries — but guard anyway. + if line_buf.last() != Some(&b'\n') { + break; + } + let raw = std::str::from_utf8(&line_buf[..n - 1]).unwrap_or("").trim(); + if raw.is_empty() { + continue; + } + let parsed: Value = match serde_json::from_str(raw) { + Ok(v) => v, + Err(_) => continue, + }; + let obj = match parsed.as_object() { + Some(o) => o.clone(), + None => continue, + }; + let line_type = obj.get("type").and_then(Value::as_str).unwrap_or(""); + match line_type { + "assistant" => { + register_assistant_node(&parsed, nodes_by_uuid); + record_evidence_from_line(evidence, &parsed); + record_explicit_relationship_evidence(evidence, &obj); + if let Some(mid) = obj + .get("message") + .and_then(|m| m.get("id")) + .and_then(Value::as_str) + { + last_assistant_message_id = Some(mid.to_string()); + } + } + "user" => { + // Match the main-loop classification: a row is a real + // user-prompt root only when it isn't a harness task + // notification AND carries plain user text. See #439 for + // the task-notification gate and #433 for why we tag + // the LineNode for the parent-chain walker. + let is_user_prompt = !is_task_notification(&obj) + && extract_plain_user_text_from_obj(&obj).is_some_and(|s| !s.is_empty()); + register_user_node(&parsed, nodes_by_uuid, is_user_prompt); + record_evidence_from_line(evidence, &parsed); + record_explicit_relationship_evidence(evidence, &obj); + record_resume_marker(evidence, &obj); + let mut harvested: Vec = Vec::new(); + next_event_index = collect_tool_result_events( + &obj, + &mut harvested, + tool_result_counters, + next_event_index, + ); + } + "system" + if build_claude_system_tool_result_event( + &obj, + tool_result_counters, + next_event_index, + ) + .is_some() => + { + next_event_index += 1; + } + _ => {} + } + } + Ok(PrescanOutput { + last_assistant_message_id, + next_event_index, + }) +} + +fn record_root_incremental( + out: &mut Vec<(u64, SessionRelationshipRecord)>, + seen: &mut HashSet, + session_id: &str, + ts: Option<&str>, + line_offset: u64, + file_session_id: Option<&str>, +) { + let canonical = file_session_id.unwrap_or(session_id).to_string(); + if !seen.insert(canonical.clone()) { + return; + } + let mut row = SessionRelationshipRecord { + v: 1, + source: RelationshipSourceKind::ClaudeCode, + session_id: canonical, + related_session_id: None, + relationship_type: RelationshipType::Root, + ts: None, + source_session_id: None, + source_version: None, + parent_tool_use_id: None, + agent_id: None, + subagent_type: None, + description: None, + }; + if let Some(t) = ts { + if !t.is_empty() { + row.ts = Some(t.to_string()); + } + } + out.push((line_offset, row)); +} + +/// Owns the mutable working state threaded through the Claude incremental +/// parser's streaming loop, mirroring the codex decomposition's +/// `CodexParseState`. The driver (`run_incremental`) constructs one, dispatches +/// each parsed line to `handle_assistant` / `handle_user` / `handle_system`, +/// then assembles the result from this state's accumulated fields after the +/// loop closes. Field names and initializers reproduce the prior inline locals +/// verbatim. +struct ClaudeParseState { + file_session_id: Option, + evidence: ClaudeRelationshipEvidence, + nodes_by_uuid: HashMap, + invocation_cache: HashMap>, + tool_result_counters: HashMap, + next_event_index: u64, + last_assistant_message_id: Option, + // -1 sentinel: resume marker came from the prescan (definitely emit). + // u64::MAX sentinel: no resume marker yet seen. + // Otherwise: byte offset of the line that first set the marker on this pass. + resume_marker_offset: u64, + current_user_text: String, + working: HashMap, + order: Vec, + message_id_first_offset: HashMap, + // Legacy file-order map kept as a fallback when the assistant row + // lacks a `uuid` or its parent chain doesn't terminate at a known + // user prompt. The preferred lookup is `user_text_by_uuid` walked + // via `nearest_user_prompt_root` (#433). + user_text_by_message_id: HashMap, + // User-prompt text keyed by the user line's own `uuid` — read by the + // parent-chain walker during turn classification. Populated only for + // real user prompts (task notifications excluded; empty bodies + // excluded). + user_text_by_uuid: HashMap, + errored_tool_use_ids: HashSet, + replacement_meta_by_tool_use_id: HashMap, + // Slash-command triad detection (#438) needs a flat slice of user-typed + // rows to look up the parent-UUID chain shape. We accumulate only the + // minimal field set the detector reads (`type`, `uuid`, `parentUuid`, + // `message.content`) so memory stays bounded — three rows per triad, + // and only user-typed rows are stored. Detection runs once after the + // streaming loop closes; the resulting `skill_uuids` set is consulted + // by `apply_classification` to override the activity to `Skill`. + user_rows_for_triad: Vec>, + events: Vec<(u64, CompactionEvent)>, + pending_user_content: Vec<(u64, ContentRecord)>, + pending_tool_result_events: Vec<(u64, ToolResultEventRecord)>, + pending_relationships: Vec<(u64, SessionRelationshipRecord)>, + pending_user_turns: Vec<(u64, UserTurnRecord)>, + seen_root_session_ids: HashSet, + seen_explicit_relationship_ids: HashSet, + pending_user_turn_inc_idx: Option, +} + +impl ClaudeParseState { + /// Initialize working state, reproducing the prior inline initializers + /// verbatim — including the `prescan_nodes` wiring (run only when + /// `start_offset > 0`) and the `resume_marker_offset` sentinel derivation + /// off `evidence.has_resume_marker`. + fn new( + path: &Path, + start_offset: u64, + file_session_id: Option, + last_user_text: Option<&str>, + ) -> std::io::Result { + let mut evidence = new_evidence(file_session_id.clone()); + + let mut nodes_by_uuid: HashMap = HashMap::new(); + let mut tool_result_counters: HashMap = HashMap::new(); + let mut next_event_index: u64 = 0; + let mut last_assistant_message_id: Option = None; + + if start_offset > 0 { + let pre = prescan_nodes( + path, + start_offset, + &mut nodes_by_uuid, + &mut evidence, + &mut tool_result_counters, + )?; + last_assistant_message_id = pre.last_assistant_message_id; + next_event_index = pre.next_event_index; + } + + // -1 sentinel: resume marker came from the prescan (definitely emit). + // u64::MAX sentinel: no resume marker yet seen. + // Otherwise: byte offset of the line that first set the marker on this pass. + let resume_marker_offset: u64 = if evidence.has_resume_marker { + 0 + } else { + u64::MAX + }; + + Ok(Self { + file_session_id, + evidence, + nodes_by_uuid, + invocation_cache: HashMap::new(), + tool_result_counters, + next_event_index, + last_assistant_message_id, + resume_marker_offset, + current_user_text: last_user_text.map(str::to_string).unwrap_or_default(), + working: HashMap::new(), + order: Vec::new(), + message_id_first_offset: HashMap::new(), + user_text_by_message_id: HashMap::new(), + user_text_by_uuid: HashMap::new(), + errored_tool_use_ids: HashSet::new(), + replacement_meta_by_tool_use_id: HashMap::new(), + user_rows_for_triad: Vec::new(), + events: Vec::new(), + pending_user_content: Vec::new(), + pending_tool_result_events: Vec::new(), + pending_relationships: Vec::new(), + pending_user_turns: Vec::new(), + seen_root_session_ids: HashSet::new(), + seen_explicit_relationship_ids: HashSet::new(), + pending_user_turn_inc_idx: None, + }) + } + + fn handle_assistant( + &mut self, + parsed: &Value, + obj: &serde_json::Map, + line_start_offset: u64, + ) { + let mid = obj + .get("message") + .and_then(|m| m.get("id")) + .and_then(Value::as_str) + .map(str::to_string); + if let Some(ref mid_str) = mid { + if let Some(idx) = self.pending_user_turn_inc_idx { + if !self.message_id_first_offset.contains_key(mid_str) { + self.pending_user_turns[idx].1.following_message_id = Some(mid_str.clone()); + self.pending_user_turn_inc_idx = None; + } + } + self.message_id_first_offset + .entry(mid_str.clone()) + .or_insert(line_start_offset); + self.user_text_by_message_id + .entry(mid_str.clone()) + .or_insert_with(|| self.current_user_text.clone()); + self.last_assistant_message_id = Some(mid_str.clone()); + } + let session_id = string_field(obj, SESSION_ID_KEYS, false); + let timestamp = string_field(obj, TIMESTAMP_KEYS, false); + if let Some(ref sid) = session_id { + if !sid.is_empty() { + record_root_incremental( + &mut self.pending_relationships, + &mut self.seen_root_session_ids, + sid, + timestamp.as_deref(), + line_start_offset, + self.file_session_id.as_deref(), + ); + collect_explicit_claude_relationships_incremental( + obj, + &mut self.evidence, + &mut self.pending_relationships, + &mut self.seen_explicit_relationship_ids, + self.file_session_id.as_deref().unwrap_or(sid.as_str()), + timestamp.as_deref(), + line_start_offset, + ); + } + } + record_evidence_from_line(&mut self.evidence, parsed); + ingest_assistant_record( + parsed, + obj, + &mut self.working, + &mut self.order, + &mut self.nodes_by_uuid, + ); + } + + fn handle_user( + &mut self, + parsed: &Value, + obj: &serde_json::Map, + line_start_offset: u64, + counter: &C, + capture_content: bool, + ) { + // Slash-command triad detector (#438) keeps a slim copy of + // every user-typed row so a post-loop pass can find the + // caveat → invocation → stdout chain shape. We clone the + // row before the rest of the branch consumes it; the + // detector only reads four fields so memory stays modest + // (one entry per user row, dropped at function exit). + self.user_rows_for_triad.push(obj.clone()); + // Harness-injected `` rows share the user + // envelope but represent system events, not real prompts. + // Detecting them here keeps them out of `current_user_text` + // (so the classifier doesn't get task-notification text as + // "user intent") and out of `pending_user_turns` (so + // user-turn aggregates aren't inflated). Side effects like + // session-relationship discovery still run because those + // are independent of "is this a real user prompt". See + // AgentWorkforce/burn#439. + let task_notification = is_task_notification(obj); + let user_text = if task_notification { + None + } else { + extract_plain_user_text_from_obj(obj).filter(|s| !s.is_empty()) + }; + let is_user_prompt = user_text.is_some(); + register_user_node(parsed, &mut self.nodes_by_uuid, is_user_prompt); + if let Some(ref text) = user_text { + self.current_user_text = text.clone(); + // Index by the user line's UUID for the parent-chain + // walker (#433). Falls back to no-op when the row + // lacks a `uuid`, in which case file-order remains + // the only association mechanism for downstream + // assistants. + if let Some(uuid) = obj.get("uuid").and_then(Value::as_str) { + if !uuid.is_empty() { + self.user_text_by_uuid + .entry(uuid.to_string()) + .or_insert_with(|| text.clone()); + } + } + } + collect_errored_tool_use_ids(obj, &mut self.errored_tool_use_ids); + collect_replacement_meta(obj, &mut self.replacement_meta_by_tool_use_id); + let session_id = string_field(obj, SESSION_ID_KEYS, false); + let timestamp = string_field(obj, TIMESTAMP_KEYS, false); + if let Some(ref sid) = session_id { + if !sid.is_empty() { + record_root_incremental( + &mut self.pending_relationships, + &mut self.seen_root_session_ids, + sid, + timestamp.as_deref(), + line_start_offset, + self.file_session_id.as_deref(), + ); + collect_explicit_claude_relationships_incremental( + obj, + &mut self.evidence, + &mut self.pending_relationships, + &mut self.seen_explicit_relationship_ids, + self.file_session_id.as_deref().unwrap_or(sid.as_str()), + timestamp.as_deref(), + line_start_offset, + ); + } + } + record_evidence_from_line(&mut self.evidence, parsed); + let had_resume_before = self.evidence.has_resume_marker; + record_resume_marker(&mut self.evidence, obj); + if !had_resume_before && self.evidence.has_resume_marker { + self.resume_marker_offset = line_start_offset; + } + let mut harvested: Vec = Vec::new(); + self.next_event_index = collect_tool_result_events( + obj, + &mut harvested, + &mut self.tool_result_counters, + self.next_event_index, + ); + for ev in harvested { + self.pending_tool_result_events + .push((line_start_offset, ev)); + } + if !task_notification { + if let Some(record) = + build_user_turn_record(obj, self.last_assistant_message_id.as_deref(), counter) + { + let idx = self.pending_user_turns.len(); + self.pending_user_turns.push((line_start_offset, record)); + self.pending_user_turn_inc_idx = Some(idx); + } + } + if capture_content { + for c in extract_user_content(obj) { + self.pending_user_content.push((line_start_offset, c)); + } + } + } + + fn handle_system(&mut self, obj: &serde_json::Map, line_start_offset: u64) { + if obj.get("subtype").and_then(Value::as_str) == Some("compact_boundary") { + let session_id = string_field(obj, SESSION_ID_KEYS, false).unwrap_or_default(); + let ts = string_field(obj, TIMESTAMP_KEYS, false).unwrap_or_default(); + if !session_id.is_empty() { + let mut ev = CompactionEvent { + v: 1, + source: SourceKind::ClaudeCode, + session_id, + ts, + preceding_message_id: None, + tokens_before_compact: None, + }; + if let Some(ref last) = self.last_assistant_message_id { + ev.preceding_message_id = Some(last.clone()); + } + self.events.push((line_start_offset, ev)); + } + } + if let Some(ev) = build_claude_system_tool_result_event( + obj, + &mut self.tool_result_counters, + self.next_event_index, + ) { + self.pending_tool_result_events + .push((line_start_offset, ev)); + self.next_event_index += 1; + } + } +} + +pub(super) fn run_incremental( + path: &Path, + options: &ParseIncrementalOptions, + counter: &C, + emit_in_progress: bool, +) -> std::io::Result { + let start_offset = options.start_offset.unwrap_or(0); + let content_mode = options.content_mode.unwrap_or(ContentStoreMode::Off); + let capture_content = matches!(content_mode, ContentStoreMode::Full); + + let file_session_id = derive_file_session_id_from_parts( + options.file_session_id.as_deref(), + options.session_path.as_deref(), + ); + + let metadata = std::fs::metadata(path)?; + let size = metadata.len(); + if start_offset >= size { + return Ok(ParseIncrementalResult { + turns: Vec::new(), + content: Vec::new(), + events: Vec::new(), + relationships: Vec::new(), + tool_result_events: Vec::new(), + user_turns: Vec::new(), + request_id_lookup: RequestIdLookup::new(), + end_offset: start_offset, + last_user_text: options.last_user_text.clone().unwrap_or_default(), + evidence: new_evidence(file_session_id), + }); + } + + let mut state = ClaudeParseState::new( + path, + start_offset, + file_session_id, + options.last_user_text.as_deref(), + )?; + + let mut file = File::open(path)?; + file.seek(SeekFrom::Start(start_offset))?; + // Stream from `start_offset` line-by-line. The previous implementation + // allocated `Vec::with_capacity((size - start_offset) as usize)` and + // `read_to_end` into it — for a multi-GB session this was a multi-GB + // up-front allocation. With BufReader + `read_until` only the longest + // single line stays resident. + let mut reader = BufReader::new(file); + let mut line_buf: Vec = Vec::new(); + let mut cursor_offset: u64 = start_offset; // position past last complete \n + loop { + line_buf.clear(); + let n = reader.read_until(b'\n', &mut line_buf)?; + if n == 0 { + break; + } + // Drop trailing partial lines — the next incremental call resumes + // from `cursor_offset`, which we only advance past complete `\n`. + // `emit_in_progress` runs from the single-shot full-parse entry where + // there is no next call, so a final line without a trailing `\n` + // (truncated write, unflushed `\n`) must still be processed; the old + // `ParseState` path used `read_line` and surfaced it. + let has_newline = line_buf.last() == Some(&b'\n'); + if !has_newline && !emit_in_progress { + break; + } + let line_start_offset = cursor_offset; + let line_end_offset = cursor_offset + n as u64; + let body_end = if has_newline { n - 1 } else { n }; + let trimmed = std::str::from_utf8(&line_buf[..body_end]) + .unwrap_or("") + .trim(); + // Single-shot callers have no next pass, so a final partial line still + // needs to bump the cursor past its body — `end_offset = cursor_offset` + // is what the per-record offset filters compare against below. + if has_newline || emit_in_progress { + cursor_offset = line_end_offset; + } + if trimmed.is_empty() { + continue; + } + let parsed: Value = match serde_json::from_str(trimmed) { + Ok(v) => v, + Err(_) => continue, + }; + let obj = match parsed.as_object() { + Some(o) => o.clone(), + None => continue, + }; + let line_type = obj.get("type").and_then(Value::as_str).unwrap_or(""); + match line_type { + "assistant" => state.handle_assistant(&parsed, &obj, line_start_offset), + "user" => state.handle_user(&parsed, &obj, line_start_offset, counter, capture_content), + "system" => state.handle_system(&obj, line_start_offset), + _ => {} + } + } + + // Move the accumulated working state out of `state` so the post-loop + // assembly below reads the same locals the prior inline body did. + let ClaudeParseState { + file_session_id: _, + evidence, + nodes_by_uuid, + mut invocation_cache, + tool_result_counters: _, + next_event_index: _, + last_assistant_message_id: _, + resume_marker_offset, + current_user_text, + working, + order, + message_id_first_offset, + user_text_by_message_id, + user_text_by_uuid, + errored_tool_use_ids, + replacement_meta_by_tool_use_id, + user_rows_for_triad, + events, + pending_user_content, + pending_tool_result_events, + pending_relationships, + pending_user_turns, + seen_root_session_ids: _, + seen_explicit_relationship_ids: _, + pending_user_turn_inc_idx: _, + } = state; + + // end_offset = byte position of the earliest in-progress messageId, or + // cursor_offset (= position past the last complete newline) when all + // messages are complete. In `emit_in_progress` mode (the full + // non-incremental parse) we keep cursor_offset and emit every turn so the + // result matches the single-shot ParseResult contract: callers want every + // record we saw, including trailing in-progress assistants. + let end_offset = if emit_in_progress { + cursor_offset + } else { + let mut earliest_incomplete: Option = None; + for id in &order { + let w = match working.get(id) { + Some(w) => w, + None => continue, + }; + if w.stop_reason.is_none() { + if let Some(off) = message_id_first_offset.get(id) { + if earliest_incomplete.is_none_or(|e| *off < e) { + earliest_incomplete = Some(*off); + } + } + } + } + earliest_incomplete.unwrap_or(cursor_offset) + }; + + // Slash-command triad detection (#438). Run once over the accumulated + // user rows; the resulting set of triad UUIDs (caveat + invocation + + // stdout) is consulted by `apply_classification` to override the + // assistant turn's activity to `Skill` when its parent-chain root + // lands on any one of the three triad rows. Token attribution stays + // on the underlying turn's `usage` — the synthetic `Skill` label is + // a view, not a billing reattribution. + let mut skill_uuids: HashSet = HashSet::new(); + for triad in detect_slash_triads(&user_rows_for_triad) { + for idx in [triad.caveat_idx, triad.invocation_idx, triad.stdout_idx] { + if let Some(uuid) = user_rows_for_triad + .get(idx) + .and_then(|r| r.get("uuid")) + .and_then(Value::as_str) + { + if !uuid.is_empty() { + skill_uuids.insert(uuid.to_string()); + } + } + } + } + // Detector input is no longer needed; drop the cloned rows so we + // don't carry them past the emission loop. + drop(user_rows_for_triad); + + // Emit completed turns. In-progress messages (no stop_reason) are deferred + // — `end_offset` already backs up to before their first byte so the next + // call re-reads them. `emit_in_progress` opts the non-incremental path out + // of that skip so it emits every working record. + let mut turns: Vec = Vec::new(); + let mut assistant_pending: Vec<(u64, usize, ContentRecord)> = Vec::new(); + for (i, id) in order.iter().enumerate() { + let w = match working.get(id) { + Some(w) => w, + None => continue, + }; + if !emit_in_progress && w.stop_reason.is_none() { + continue; + } + let tool_calls = extract_tool_calls( + &w.blocks, + &errored_tool_use_ids, + Some(&replacement_meta_by_tool_use_id), + ); + let files_touched = extract_files_touched(&tool_calls); + let subagent = resolve_subagent(w, &nodes_by_uuid, &mut invocation_cache); + + let mut record = TurnRecord { + v: 1, + source: SourceKind::ClaudeCode, + session_id: w.session_id.clone(), + session_path: options.session_path.clone(), + message_id: w.message_id.clone(), + turn_index: i as u64, + ts: w.first_ts.clone(), + model: w.model.clone(), + project: None, + project_key: None, + usage: w.usage.clone(), + tool_calls: tool_calls.clone(), + files_touched: if files_touched.is_empty() { + None + } else { + Some(files_touched) + }, + subagent, + stop_reason: w + .stop_reason + .as_deref() + .map(|s| StopReason::from_wire(s).unwrap_or(StopReason::Silent)), + activity: None, + retries: None, + has_edits: None, + fidelity: Some(build_claude_fidelity(&w.usage_coverage)), + }; + if let Some(ref cwd) = w.cwd { + let resolved = resolve_project(cwd); + record.project = Some(resolved.project); + record.project_key = resolved.project_key; + } + apply_classification( + &mut record, + w, + &user_text_by_message_id, + &user_text_by_uuid, + &nodes_by_uuid, + &errored_tool_use_ids, + &skill_uuids, + ); + turns.push(record); + + if capture_content { + let msg_offset = *message_id_first_offset.get(&w.message_id).unwrap_or(&0); + for (sub, r) in extract_assistant_content(w).into_iter().enumerate() { + assistant_pending.push((msg_offset, sub + 1, r)); + } + } + } + + // Filter content by end_offset and interleave by source-line offset. + // appendContent has no row-level dedup, so we MUST drop rows past + // end_offset — the next call will re-read those bytes and re-emit them. + let mut content: Vec = Vec::new(); + if capture_content { + let mut merged: Vec<(u64, usize, ContentRecord)> = Vec::new(); + for (off, rec) in pending_user_content.into_iter() { + if off < end_offset { + merged.push((off, 0, rec)); + } + } + for (off, sub, rec) in assistant_pending.into_iter() { + if off < end_offset { + merged.push((off, sub, rec)); + } + } + merged.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); + content = merged.into_iter().map(|(_, _, r)| r).collect(); + } + + let mut emitted_events: Vec = events + .into_iter() + .filter(|(off, _)| *off < end_offset) + .map(|(_, ev)| ev) + .collect(); + annotate_compaction_events(&mut emitted_events, &turns); + + let mut emitted_relationships: Vec = pending_relationships + .into_iter() + .filter(|(off, _)| *off < end_offset) + .map(|(_, r)| r) + .collect(); + collect_subagent_relationships(&turns, &mut emitted_relationships); + if resume_marker_offset < end_offset { + emit_local_continuation_from_resume(&mut emitted_relationships, &evidence); + } + annotate_relationships_with_evidence(&mut emitted_relationships, &evidence); + + let mut emitted_tool_result_events: Vec = pending_tool_result_events + .into_iter() + .filter(|(off, _)| *off < end_offset) + .map(|(_, ev)| ev) + .collect(); + annotate_spawn_events(&mut emitted_tool_result_events, &turns); + + let emitted_user_turns: Vec = pending_user_turns + .into_iter() + .filter(|(off, _)| *off < end_offset) + .map(|(_, ut)| ut) + .collect(); + + // Build the `(source, session_id, message_id) -> requestId` lookup + // for every emitted turn. We only walk turns the run actually emits + // (in-progress assistant rows are filtered out above) so the lookup + // entries always correspond to an outbound `TurnRecord`. See issue + // #434. + let mut request_id_lookup = RequestIdLookup::new(); + for t in &turns { + if let Some(w) = working.get(&t.message_id) { + if let Some(req) = w.request_id.as_ref() { + if !req.is_empty() { + request_id_lookup.insert(TurnKey::for_turn(t), req.clone()); + } + } + } + } + + Ok(ParseIncrementalResult { + turns, + content, + events: emitted_events, + relationships: emitted_relationships, + tool_result_events: emitted_tool_result_events, + user_turns: emitted_user_turns, + request_id_lookup, + end_offset, + last_user_text: current_user_text, + evidence, + }) +} diff --git a/crates/relayburn-sdk/src/reader/codex.rs b/crates/relayburn-sdk/src/reader/codex.rs index 8021fd8e..0bcf97e8 100644 --- a/crates/relayburn-sdk/src/reader/codex.rs +++ b/crates/relayburn-sdk/src/reader/codex.rs @@ -10,25 +10,22 @@ //! sizing. The Rust port uses [`HeuristicCounter`] (bytes/4) — see #246 for //! the cl100k hookup. -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeSet, HashMap}; use std::fs::File; -use std::io::{BufRead, BufReader, Read, Seek, SeekFrom}; +use std::io::{BufReader, Read, Seek, SeekFrom}; use std::path::Path; use serde_json::Value; -use crate::reader::classifier::{classify_activity, ClassificationInput}; use crate::reader::fidelity::classify_fidelity; use crate::reader::git::ProjectResolver; -use crate::reader::hash::{args_hash, content_hash}; +use crate::reader::hash::content_hash; use crate::reader::types::{ - CompactionEvent, ContentKind, ContentRecord, ContentRole, ContentStoreMode, ContentToolResult, - ContentToolUse, Coverage, Fidelity, RelationshipSourceKind, RelationshipType, - SessionRelationshipRecord, SourceKind, ToolCall, ToolResultEventRecord, ToolResultEventSource, - ToolResultStatus, TurnRecord, Usage, UsageGranularity, UserTurnBlock, UserTurnBlockKind, - UserTurnRecord, + CompactionEvent, ContentRecord, ContentStoreMode, Coverage, Fidelity, RelationshipSourceKind, + RelationshipType, SessionRelationshipRecord, SourceKind, ToolCall, ToolResultEventRecord, + ToolResultStatus, TurnRecord, Usage, UsageGranularity, UserTurnBlock, UserTurnRecord, }; -use crate::reader::user_turn::{HeuristicCounter, UserTurnTokenizer}; +use crate::reader::user_turn::{resolve_token_counter, UserTurnTokenizer}; // --------------------------------------------------------------------------- // Public surface @@ -213,21 +210,21 @@ pub fn parse_codex_session_incremental( // --------------------------------------------------------------------------- #[derive(Debug, Clone, Default)] -struct UserTurnSlot { - blocks: Vec, - preceding_message_id: Option, - ts: String, +pub(in crate::reader::codex) struct UserTurnSlot { + pub(in crate::reader::codex) blocks: Vec, + pub(in crate::reader::codex) preceding_message_id: Option, + pub(in crate::reader::codex) ts: String, } impl UserTurnSlot { - fn from_persisted(p: &PersistedUserTurnSlot) -> Self { + pub(in crate::reader::codex) fn from_persisted(p: &PersistedUserTurnSlot) -> Self { Self { blocks: p.blocks.clone(), preceding_message_id: p.preceding_message_id.clone(), ts: p.ts.clone(), } } - fn to_persisted(&self) -> PersistedUserTurnSlot { + pub(in crate::reader::codex) fn to_persisted(&self) -> PersistedUserTurnSlot { PersistedUserTurnSlot { blocks: self.blocks.clone(), preceding_message_id: self.preceding_message_id.clone(), @@ -237,51 +234,54 @@ impl UserTurnSlot { } #[derive(Debug, Clone)] -struct SpawnCallInfo { - call_id: String, - ts: String, - subagent_type: Option, - description: Option, - spawned_agent_id: Option, - emitted: bool, +pub(in crate::reader::codex) struct SpawnCallInfo { + pub(in crate::reader::codex) call_id: String, + pub(in crate::reader::codex) ts: String, + pub(in crate::reader::codex) subagent_type: Option, + pub(in crate::reader::codex) description: Option, + pub(in crate::reader::codex) spawned_agent_id: Option, + pub(in crate::reader::codex) emitted: bool, } #[derive(Debug, Clone)] -struct OpenTurn { - turn_id: String, - ts: String, - model: String, - project: Option, - start_cumulative: CumulativeUsage, - tool_calls: Vec, - seen_call_ids: BTreeSet, - files_touched: BTreeSet, - user_text: String, - assistant_text: String, - errored_call_ids: BTreeSet, - content: Vec, - pending_tool_result_events: Vec, - pending_relationships: Vec, - spawn_calls: HashMap, - usage_observed: bool, -} - -struct FinalizedTurn { - turn_id: String, - ts: String, - model: String, - project: Option, - tool_calls: Vec, - files_touched: Vec, - user_text: String, - assistant_text: String, - errored_call_ids: BTreeSet, - content: Vec, - usage: Usage, - fidelity: Fidelity, -} - -fn finalize_turn(open: OpenTurn, cumulative: &CumulativeUsage) -> FinalizedTurn { +pub(in crate::reader::codex) struct OpenTurn { + pub(in crate::reader::codex) turn_id: String, + pub(in crate::reader::codex) ts: String, + pub(in crate::reader::codex) model: String, + pub(in crate::reader::codex) project: Option, + pub(in crate::reader::codex) start_cumulative: CumulativeUsage, + pub(in crate::reader::codex) tool_calls: Vec, + pub(in crate::reader::codex) seen_call_ids: BTreeSet, + pub(in crate::reader::codex) files_touched: BTreeSet, + pub(in crate::reader::codex) user_text: String, + pub(in crate::reader::codex) assistant_text: String, + pub(in crate::reader::codex) errored_call_ids: BTreeSet, + pub(in crate::reader::codex) content: Vec, + pub(in crate::reader::codex) pending_tool_result_events: Vec, + pub(in crate::reader::codex) pending_relationships: Vec, + pub(in crate::reader::codex) spawn_calls: HashMap, + pub(in crate::reader::codex) usage_observed: bool, +} + +pub(in crate::reader::codex) struct FinalizedTurn { + pub(in crate::reader::codex) turn_id: String, + pub(in crate::reader::codex) ts: String, + pub(in crate::reader::codex) model: String, + pub(in crate::reader::codex) project: Option, + pub(in crate::reader::codex) tool_calls: Vec, + pub(in crate::reader::codex) files_touched: Vec, + pub(in crate::reader::codex) user_text: String, + pub(in crate::reader::codex) assistant_text: String, + pub(in crate::reader::codex) errored_call_ids: BTreeSet, + pub(in crate::reader::codex) content: Vec, + pub(in crate::reader::codex) usage: Usage, + pub(in crate::reader::codex) fidelity: Fidelity, +} + +pub(in crate::reader::codex) fn finalize_turn( + open: OpenTurn, + cumulative: &CumulativeUsage, +) -> FinalizedTurn { let usage = Usage { input: (cumulative.input - open.start_cumulative.input).max(0) as u64, output: (cumulative.output - open.start_cumulative.output).max(0) as u64, @@ -328,914 +328,11 @@ fn build_codex_fidelity(usage_observed: bool) -> Fidelity { } } -// --------------------------------------------------------------------------- -// Core parser loop -// --------------------------------------------------------------------------- - -#[derive(Debug, Clone)] -struct Pending { - offset: u64, - record: T, -} - -fn parse_codex_buffer( - mut reader: R, - start_offset: u64, - options: &ParseCodexIncrementalOptions, - project_resolver: &ProjectResolver, -) -> std::io::Result { - let capture_content = matches!(options.content_mode, Some(ContentStoreMode::Full)); - // Validated by `resolve_token_counter` at the public entry point. - let counter = HeuristicCounter; - - let resume = options.resume.as_ref(); - let mut session_id = resume.map(|r| r.session_id.clone()).unwrap_or_default(); - let mut session_cwd: Option = resume.and_then(|r| r.session_cwd.clone()); - let mut turn_contexts: HashMap = - resume.map(|r| r.turn_contexts.clone()).unwrap_or_default(); - let mut cumulative = resume.map(|r| r.cumulative.clone()).unwrap_or_default(); - - let mut open_turn: Option = None; - let mut pending_user_text = String::new(); - let mut pending_content: Vec = Vec::new(); - let mut finalized: Vec = Vec::new(); - - let mut user_turn_slot: UserTurnSlot = resume - .and_then(|r| r.user_turn_slot.as_ref()) - .map(UserTurnSlot::from_persisted) - .unwrap_or_default(); - let mut user_turns: Vec = Vec::new(); - - let mut root_session_emitted = resume.map(|r| r.root_session_emitted).unwrap_or(false); - let mut seen_session_meta_keys: BTreeSet = resume - .map(|r| r.session_meta_relationship_keys.iter().cloned().collect()) - .unwrap_or_default(); - let mut next_event_index = resume.map(|r| r.next_event_index).unwrap_or(0); - let mut tool_result_counters: HashMap = resume - .map(|r| r.tool_result_counters.clone()) - .unwrap_or_default(); - let mut last_completed_turn: Option = - resume.and_then(|r| r.last_completed_turn.clone()); - - let mut pending_tool_result_events: Vec> = Vec::new(); - let mut pending_relationships: Vec> = Vec::new(); - let mut pending_compactions: Vec> = Vec::new(); - - let mut committed_end_offset = start_offset; - let mut committed_cumulative = cumulative.clone(); - let mut committed_session_id = session_id.clone(); - let mut committed_session_cwd = session_cwd.clone(); - let mut committed_turn_contexts = turn_contexts.clone(); - let mut committed_finalized_count: usize = 0; - let mut committed_user_turns_count: usize = 0; - let mut committed_user_turn_slot = user_turn_slot.clone(); - let mut committed_root_session_emitted = root_session_emitted; - let mut committed_seen_session_meta_keys = seen_session_meta_keys.clone(); - let mut committed_next_event_index = next_event_index; - let mut committed_tool_result_counters = tool_result_counters.clone(); - let mut committed_last_completed_turn = last_completed_turn.clone(); - - let mut line_buf: Vec = Vec::new(); - let mut current_offset: u64 = start_offset; - loop { - line_buf.clear(); - let n = reader.read_until(b'\n', &mut line_buf)?; - if n == 0 { - break; - } - // Drop trailing partial lines — the next incremental call resumes - // from the committed end offset, which only advances past `\n`. - if line_buf.last() != Some(&b'\n') { - break; - } - let line_end_offset = current_offset + n as u64; - current_offset = line_end_offset; - let text = std::str::from_utf8(&line_buf[..n - 1]).unwrap_or("").trim(); - if text.is_empty() { - continue; - } - let parsed: Value = match serde_json::from_str(text) { - Ok(v) => v, - Err(_) => continue, - }; - if !parsed.is_object() { - continue; - } - let rec_type = parsed.get("type").and_then(|v| v.as_str()).unwrap_or(""); - let rec_timestamp = parsed - .get("timestamp") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let payload = match parsed.get("payload") { - Some(p) if p.is_object() => p, - _ => continue, - }; - - match rec_type { - "session_meta" => { - if let Some(id) = session_meta_payload_id(payload) { - session_id = id; - } - if let Some(cwd) = payload.get("cwd").and_then(|v| v.as_str()) { - session_cwd = Some(cwd.to_string()); - if let Some(open) = open_turn.as_mut() { - if open.project.is_none() { - open.project = Some(cwd.to_string()); - } - } - } - if !session_id.is_empty() && !root_session_emitted { - root_session_emitted = true; - let ts = payload - .get("timestamp") - .and_then(|v| v.as_str()) - .unwrap_or(rec_timestamp); - pending_relationships.push(Pending { - offset: line_end_offset, - record: build_root_relationship(&session_id, ts, payload), - }); - } - if !session_id.is_empty() { - for row in build_session_meta_relationships(&session_id, payload, rec_timestamp) - { - let key = codex_relationship_key(&row); - if seen_session_meta_keys.contains(&key) { - continue; - } - seen_session_meta_keys.insert(key); - pending_relationships.push(Pending { - offset: line_end_offset, - record: row, - }); - } - } - continue; - } - "turn_context" => { - let ctx = CodexTurnContext { - turn_id: payload - .get("turn_id") - .and_then(|v| v.as_str()) - .map(str::to_string), - cwd: payload - .get("cwd") - .and_then(|v| v.as_str()) - .map(str::to_string), - model: payload - .get("model") - .and_then(|v| v.as_str()) - .map(str::to_string), - }; - if let Some(tid) = ctx.turn_id.clone() { - turn_contexts.insert(tid.clone(), ctx.clone()); - if let Some(open) = open_turn.as_mut() { - if open.turn_id == tid { - if open.model.is_empty() { - if let Some(m) = ctx.model.as_deref() { - open.model = m.to_string(); - } - } - if open.project.is_none() { - if let Some(c) = ctx.cwd.as_deref() { - open.project = Some(c.to_string()); - } - } - } - } - } - continue; - } - "compacted" => { - if !session_id.is_empty() { - pending_compactions.push(Pending { - offset: line_end_offset, - record: build_codex_compaction_event( - &session_id, - rec_timestamp, - last_completed_turn.as_ref(), - ), - }); - } - continue; - } - "event_msg" => { - let pl_type = payload.get("type").and_then(|v| v.as_str()).unwrap_or(""); - match pl_type { - "token_count" => { - if let Some(total) = payload.get("info").and_then(|i| { - if i.is_null() { - None - } else { - i.get("total_token_usage") - } - }) { - let input_total = total - .get("input_tokens") - .and_then(|v| v.as_i64()) - .unwrap_or(0); - let cached = total - .get("cached_input_tokens") - .and_then(|v| v.as_i64()) - .unwrap_or(0); - cumulative.input = input_total - cached; - cumulative.cache_read = cached; - cumulative.output = total - .get("output_tokens") - .and_then(|v| v.as_i64()) - .unwrap_or(0); - cumulative.reasoning = total - .get("reasoning_output_tokens") - .and_then(|v| v.as_i64()) - .unwrap_or(0); - if let Some(open) = open_turn.as_mut() { - open.usage_observed = true; - } - } - } - "task_started" => { - let ts = rec_timestamp; - let turn_id = match payload.get("turn_id").and_then(|v| v.as_str()) { - Some(t) => t.to_string(), - None => continue, - }; - if let Some(open) = open_turn.take() { - finalized.push(finalize_turn(open, &cumulative)); - } - if !user_turn_slot.blocks.is_empty() { - user_turns.push(build_codex_user_turn_record( - &user_turn_slot, - &session_id, - &turn_id, - ts, - )); - } - user_turn_slot = UserTurnSlot::default(); - let ctx = turn_contexts.get(&turn_id).cloned(); - let project = ctx - .as_ref() - .and_then(|c| c.cwd.clone()) - .or_else(|| session_cwd.clone()); - let mut open = OpenTurn { - turn_id: turn_id.clone(), - ts: ts.to_string(), - model: ctx - .as_ref() - .and_then(|c| c.model.clone()) - .unwrap_or_default(), - project, - start_cumulative: cumulative.clone(), - tool_calls: vec![], - seen_call_ids: BTreeSet::new(), - files_touched: BTreeSet::new(), - user_text: std::mem::take(&mut pending_user_text), - assistant_text: String::new(), - errored_call_ids: BTreeSet::new(), - content: vec![], - pending_tool_result_events: vec![], - pending_relationships: vec![], - spawn_calls: HashMap::new(), - usage_observed: false, - }; - if capture_content && !pending_content.is_empty() { - for c in pending_content.iter_mut() { - c.message_id = turn_id.clone(); - } - open.content.append(&mut pending_content); - } - open_turn = Some(open); - } - "task_complete" => { - let payload_turn_id = payload - .get("turn_id") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let mut took: Option = None; - if let Some(open) = open_turn.as_ref() { - if open.turn_id == payload_turn_id { - took = open_turn.take(); - } - } - if let Some(mut open) = took { - // Patch isError on tool-result blocks accumulated this turn - for b in user_turn_slot.blocks.iter_mut() { - if matches!(b.kind, UserTurnBlockKind::ToolResult) { - if let Some(id) = &b.tool_use_id { - if open.errored_call_ids.contains(id) { - b.is_error = Some(true); - } - } - } - } - // Drain pending tool result events / relationships - let mut events = std::mem::take(&mut open.pending_tool_result_events); - for ev in events.iter_mut() { - if open.errored_call_ids.contains(&ev.tool_use_id) { - ev.status = ToolResultStatus::Errored; - ev.is_error = Some(true); - } else if matches!(ev.status, ToolResultStatus::Unknown) { - ev.status = ToolResultStatus::Completed; - } - } - for ev in events { - pending_tool_result_events.push(Pending { - offset: line_end_offset, - record: ev, - }); - } - let rels = std::mem::take(&mut open.pending_relationships); - for r in rels { - pending_relationships.push(Pending { - offset: line_end_offset, - record: r, - }); - } - user_turn_slot.preceding_message_id = Some(open.turn_id.clone()); - let closed = finalize_turn(open, &cumulative); - last_completed_turn = Some(CodexLastCompletedTurn { - message_id: closed.turn_id.clone(), - cache_read: closed.usage.cache_read, - }); - finalized.push(closed); - // Commit snapshot - committed_end_offset = line_end_offset; - committed_cumulative = cumulative.clone(); - committed_session_id = session_id.clone(); - committed_session_cwd = session_cwd.clone(); - committed_turn_contexts = turn_contexts.clone(); - committed_finalized_count = finalized.len(); - committed_user_turns_count = user_turns.len(); - committed_user_turn_slot = user_turn_slot.clone(); - committed_root_session_emitted = root_session_emitted; - committed_seen_session_meta_keys = seen_session_meta_keys.clone(); - committed_next_event_index = next_event_index; - committed_tool_result_counters = tool_result_counters.clone(); - committed_last_completed_turn = last_completed_turn.clone(); - } - } - "patch_apply_end" => { - if let Some(open) = open_turn.as_mut() { - let turn_id = payload.get("turn_id").and_then(|v| v.as_str()); - if turn_id != Some(open.turn_id.as_str()) { - continue; - } - let success = payload.get("success").and_then(|v| v.as_bool()); - if success == Some(false) { - if let Some(call_id) = - payload.get("call_id").and_then(|v| v.as_str()) - { - open.errored_call_ids.insert(call_id.to_string()); - } - continue; - } - if let Some(changes) = - payload.get("changes").and_then(|v| v.as_object()) - { - for file in changes.keys() { - open.files_touched.insert(file.clone()); - } - } - } - } - "exec_command_end" => { - if let Some(open) = open_turn.as_mut() { - let turn_id = payload.get("turn_id").and_then(|v| v.as_str()); - if turn_id != Some(open.turn_id.as_str()) { - continue; - } - let exit_code = payload.get("exit_code").and_then(|v| v.as_i64()); - if let (Some(code), Some(call_id)) = - (exit_code, payload.get("call_id").and_then(|v| v.as_str())) - { - if code != 0 { - open.errored_call_ids.insert(call_id.to_string()); - } - } - } - } - other if is_subagent_terminal_notification(other) => { - let call_id = match payload.get("call_id").and_then(|v| v.as_str()) { - Some(c) if !c.is_empty() => c.to_string(), - _ => continue, - }; - let entry = tool_result_counters.entry(call_id.clone()).or_insert(0); - let call_index = *entry; - *entry += 1; - let status = subagent_notification_status(payload); - let mut ev = ToolResultEventRecord { - v: 1, - source: SourceKind::Codex, - session_id: session_id.clone(), - message_id: open_turn.as_ref().map(|o| o.turn_id.clone()), - tool_use_id: call_id.clone(), - call_index: Some(call_index), - event_index: next_event_index, - ts: if rec_timestamp.is_empty() { - None - } else { - Some(rec_timestamp.to_string()) - }, - status, - event_source: ToolResultEventSource::SubagentNotification, - content_length: None, - output_bytes: None, - output_truncated: None, - content_hash: None, - is_error: matches!(status, ToolResultStatus::Errored).then_some(true), - usage: None, - usage_attribution: None, - subagent_session_id: None, - agent_id: None, - replaced_tools: None, - collapsed_calls: None, - }; - next_event_index += 1; - let spawned_id = - pick_string_field(payload, &["agent_id", "subagent_id", "session_id"]); - if let Some(sid) = spawned_id.as_ref() { - ev.agent_id = Some(sid.clone()); - ev.subagent_session_id = Some(sid.clone()); - if let Some(open) = open_turn.as_mut() { - if let Some(spawn) = open.spawn_calls.get_mut(&call_id) { - if spawn.spawned_agent_id.is_none() { - spawn.spawned_agent_id = Some(sid.clone()); - let info = spawn.clone(); - maybe_emit_spawn_relationship( - open, - &session_id, - &info, - rec_timestamp, - ); - } - } - } - } - if let Some(open) = open_turn.as_mut() { - open.pending_tool_result_events.push(ev); - } else { - pending_tool_result_events.push(Pending { - offset: line_end_offset, - record: ev, - }); - } - } - _ => {} - } - continue; - } - "response_item" => { - let item_ts = rec_timestamp; - let pl_type = payload.get("type").and_then(|v| v.as_str()).unwrap_or(""); - match pl_type { - "message" => { - let role = payload - .get("role") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let text = collect_message_text(payload, &role); - if text.is_empty() { - continue; - } - if role == "user" { - if let Some(open) = open_turn.as_mut() { - open.user_text = append_text(&open.user_text, &text); - } else { - pending_user_text = append_text(&pending_user_text, &text); - } - user_turn_slot - .blocks - .push(UserTurnBlock::text(&text, &counter)); - if user_turn_slot.ts.is_empty() && !item_ts.is_empty() { - user_turn_slot.ts = item_ts.to_string(); - } - if capture_content { - let rec = ContentRecord { - v: 1, - source: SourceKind::Codex, - session_id: session_id.clone(), - message_id: open_turn - .as_ref() - .map(|o| o.turn_id.clone()) - .unwrap_or_default(), - ts: item_ts.to_string(), - role: ContentRole::User, - kind: ContentKind::Text, - text: Some(text.clone()), - tool_use: None, - tool_result: None, - }; - push_content(&mut open_turn, &mut pending_content, rec); - } - } else if role == "assistant" { - if let Some(open) = open_turn.as_mut() { - open.assistant_text = append_text(&open.assistant_text, &text); - if capture_content { - open.content.push(ContentRecord { - v: 1, - source: SourceKind::Codex, - session_id: session_id.clone(), - message_id: open.turn_id.clone(), - ts: item_ts.to_string(), - role: ContentRole::Assistant, - kind: ContentKind::Text, - text: Some(text), - tool_use: None, - tool_result: None, - }); - } - } - } - } - "reasoning" => { - if !capture_content { - continue; - } - let Some(open) = open_turn.as_mut() else { - continue; - }; - let text = collect_reasoning_text(payload); - if !text.is_empty() { - open.content.push(ContentRecord { - v: 1, - source: SourceKind::Codex, - session_id: session_id.clone(), - message_id: open.turn_id.clone(), - ts: item_ts.to_string(), - role: ContentRole::Assistant, - kind: ContentKind::Thinking, - text: Some(text), - tool_use: None, - tool_result: None, - }); - } - } - "function_call_output" | "custom_tool_call_output" => { - let call_id = match payload.get("call_id").and_then(|v| v.as_str()) { - Some(c) => c.to_string(), - None => continue, - }; - let output = payload.get("output").cloned().unwrap_or(Value::Null); - user_turn_slot.blocks.push(UserTurnBlock::tool_result( - call_id.clone(), - &output, - None, - &counter, - )); - if user_turn_slot.ts.is_empty() && !item_ts.is_empty() { - user_turn_slot.ts = item_ts.to_string(); - } - let entry = tool_result_counters.entry(call_id.clone()).or_insert(0); - let call_index = *entry; - *entry += 1; - let initial_status = if open_turn - .as_ref() - .map(|o| o.errored_call_ids.contains(&call_id)) - .unwrap_or(false) - { - ToolResultStatus::Errored - } else { - ToolResultStatus::Unknown - }; - let measured = measure_tool_output(&output); - let mut ev = ToolResultEventRecord { - v: 1, - source: SourceKind::Codex, - session_id: session_id.clone(), - message_id: open_turn.as_ref().map(|o| o.turn_id.clone()), - tool_use_id: call_id.clone(), - call_index: Some(call_index), - event_index: next_event_index, - ts: if item_ts.is_empty() { - None - } else { - Some(item_ts.to_string()) - }, - status: initial_status, - event_source: ToolResultEventSource::FunctionCallOutput, - content_length: measured.length, - output_bytes: measured.byte_length, - // Codex doesn't carry an explicit truncation marker - // distinct from its general output; leave None until - // we have a concrete signal to flip on. - output_truncated: None, - content_hash: measured.hash, - is_error: matches!(initial_status, ToolResultStatus::Errored) - .then_some(true), - usage: None, - usage_attribution: None, - subagent_session_id: None, - agent_id: None, - replaced_tools: None, - collapsed_calls: None, - }; - next_event_index += 1; - if let Some(open) = open_turn.as_mut() { - if let Some(spawn) = open.spawn_calls.get_mut(&call_id) { - if let Some(sid) = extract_spawned_agent_id(&output) { - spawn.spawned_agent_id = Some(sid.clone()); - ev.agent_id = Some(sid.clone()); - ev.subagent_session_id = Some(sid); - } - let info = spawn.clone(); - maybe_emit_spawn_relationship(open, &session_id, &info, item_ts); - } - open.pending_tool_result_events.push(ev); - } else { - pending_tool_result_events.push(Pending { - offset: line_end_offset, - record: ev, - }); - } - if capture_content { - let rec = ContentRecord { - v: 1, - source: SourceKind::Codex, - session_id: session_id.clone(), - message_id: open_turn - .as_ref() - .map(|o| o.turn_id.clone()) - .unwrap_or_default(), - ts: item_ts.to_string(), - role: ContentRole::ToolResult, - kind: ContentKind::ToolResult, - text: None, - tool_use: None, - tool_result: Some(ContentToolResult { - tool_use_id: call_id, - content: output, - is_error: None, - }), - }; - push_content(&mut open_turn, &mut pending_content, rec); - } - } - "function_call" => { - let Some(open) = open_turn.as_mut() else { - continue; - }; - let name = match payload.get("name").and_then(|v| v.as_str()) { - Some(n) => n.to_string(), - None => continue, - }; - let call_id = match payload.get("call_id").and_then(|v| v.as_str()) { - Some(c) => c.to_string(), - None => continue, - }; - if open.seen_call_ids.contains(&call_id) { - continue; - } - open.seen_call_ids.insert(call_id.clone()); - let arg_str = payload.get("arguments").and_then(|v| v.as_str()); - let parsed_args = arg_str.and_then(safe_parse_json_object); - let hash_input = parsed_args - .clone() - .map(Value::Object) - .unwrap_or_else(|| Value::Object(Default::default())); - let target = pick_function_call_target(&name, parsed_args.as_ref()); - let call = ToolCall { - id: call_id.clone(), - name: name.clone(), - target, - args_hash: args_hash(&hash_input), - is_error: None, - edit_pre_hash: None, - edit_post_hash: None, - skill_name: None, - replaced_tools: None, - collapsed_calls: None, - }; - open.tool_calls.push(call); - if name == "spawn_agent" { - let mut info = SpawnCallInfo { - call_id: call_id.clone(), - ts: item_ts.to_string(), - subagent_type: None, - description: None, - spawned_agent_id: None, - emitted: false, - }; - if let Some(args) = parsed_args.as_ref() { - let v = Value::Object(args.clone()); - info.subagent_type = - pick_string_field(&v, &["subagent_type", "agent_type", "type"]); - info.description = - pick_string_field(&v, &["description", "task", "prompt"]); - info.spawned_agent_id = pick_string_field( - &v, - &["agent_id", "subagent_id", "session_id"], - ); - } - open.spawn_calls.insert(call_id.clone(), info.clone()); - maybe_emit_spawn_relationship(open, &session_id, &info, item_ts); - } - if capture_content { - let input = parsed_args - .map(|m| m.into_iter().collect()) - .unwrap_or_default(); - open.content.push(ContentRecord { - v: 1, - source: SourceKind::Codex, - session_id: session_id.clone(), - message_id: open.turn_id.clone(), - ts: item_ts.to_string(), - role: ContentRole::Assistant, - kind: ContentKind::ToolUse, - text: None, - tool_use: Some(ContentToolUse { - id: call_id, - name, - input, - }), - tool_result: None, - }); - } - } - "custom_tool_call" => { - let Some(open) = open_turn.as_mut() else { - continue; - }; - let name = match payload.get("name").and_then(|v| v.as_str()) { - Some(n) => n.to_string(), - None => continue, - }; - let call_id = match payload.get("call_id").and_then(|v| v.as_str()) { - Some(c) => c.to_string(), - None => continue, - }; - if open.seen_call_ids.contains(&call_id) { - continue; - } - open.seen_call_ids.insert(call_id.clone()); - let input = payload - .get("input") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let hash_input = serde_json::json!({ "input": input }); - let target = pick_custom_tool_target(&name, &input); - let call = ToolCall { - id: call_id.clone(), - name: name.clone(), - target, - args_hash: args_hash(&hash_input), - is_error: None, - edit_pre_hash: None, - edit_post_hash: None, - skill_name: None, - replaced_tools: None, - collapsed_calls: None, - }; - open.tool_calls.push(call); - if capture_content { - let mut input_map = BTreeMap::new(); - input_map.insert("input".to_string(), Value::String(input)); - open.content.push(ContentRecord { - v: 1, - source: SourceKind::Codex, - session_id: session_id.clone(), - message_id: open.turn_id.clone(), - ts: item_ts.to_string(), - role: ContentRole::Assistant, - kind: ContentKind::ToolUse, - text: None, - tool_use: Some(ContentToolUse { - id: call_id, - name, - input: input_map, - }), - tool_result: None, - }); - } - } - _ => {} - } - continue; - } - _ => continue, - } - } - - // Emit only committed turns. - let committed = &finalized[..committed_finalized_count]; - let mut turns: Vec = Vec::with_capacity(committed.len()); - let mut content_out: Vec = Vec::new(); - for (i, f) in committed.iter().enumerate() { - let mut record = TurnRecord { - v: 1, - source: SourceKind::Codex, - session_id: committed_session_id.clone(), - session_path: options.session_path.clone(), - message_id: f.turn_id.clone(), - turn_index: i as u64, - ts: f.ts.clone(), - model: f.model.clone(), - project: None, - project_key: None, - usage: f.usage.clone(), - tool_calls: f.tool_calls.clone(), - files_touched: if f.files_touched.is_empty() { - None - } else { - Some(f.files_touched.clone()) - }, - subagent: None, - stop_reason: None, - activity: None, - retries: None, - has_edits: None, - fidelity: Some(f.fidelity.clone()), - }; - if let Some(p) = f.project.as_ref() { - let resolved = project_resolver.resolve(p); - record.project = Some(resolved.project); - record.project_key = resolved.project_key; - } - let combined_text = join_nonempty(&[f.user_text.as_str(), f.assistant_text.as_str()], "\n"); - let has_failed_tool = f - .tool_calls - .iter() - .any(|tc| f.errored_call_ids.contains(&tc.id)); - let classified = classify_activity(ClassificationInput { - tool_calls: &f.tool_calls, - text: &combined_text, - has_failed_tool, - reasoning_tokens: f.usage.reasoning, - }); - record.activity = Some(classified.activity); - record.retries = Some(classified.retries); - record.has_edits = Some(classified.has_edits); - turns.push(record); - if capture_content { - content_out.extend(f.content.clone()); - } - } - - let resume = CodexResumeState { - cumulative: committed_cumulative.clone(), - session_id: committed_session_id.clone(), - session_cwd: committed_session_cwd.clone(), - turn_contexts: committed_turn_contexts.clone(), - user_turn_slot: Some(committed_user_turn_slot.to_persisted()), - root_session_emitted: committed_root_session_emitted, - session_meta_relationship_keys: committed_seen_session_meta_keys.iter().cloned().collect(), - next_event_index: committed_next_event_index, - tool_result_counters: committed_tool_result_counters.clone(), - last_completed_turn: committed_last_completed_turn.clone(), - }; - - let user_turns_out = user_turns[..committed_user_turns_count].to_vec(); - let mut events_out: Vec = Vec::new(); - for e in pending_compactions { - if e.offset <= committed_end_offset { - events_out.push(e.record); - } - } - let mut relationships_out: Vec = Vec::new(); - for r in pending_relationships { - if r.offset <= committed_end_offset { - relationships_out.push(r.record); - } - } - let mut tool_events_out: Vec = Vec::new(); - for ev in pending_tool_result_events { - if ev.offset <= committed_end_offset { - tool_events_out.push(ev.record); - } - } - - Ok(ParseCodexIncrementalResult { - turns, - content: content_out, - events: events_out, - user_turns: user_turns_out, - relationships: relationships_out, - tool_result_events: tool_events_out, - end_offset: committed_end_offset, - resume, - }) -} - // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- -/// Resolves the requested tokenizer to a concrete counter. `None` and -/// `Some(Heuristic)` map to [`HeuristicCounter`]; `Some(Cl100k)` is rejected -/// with an explicit error until the cl100k counter is wired up (see #246) so -/// callers don't silently get bytes/4 sizing when they asked for cl100k. -fn resolve_token_counter( - tokenizer: Option, -) -> std::io::Result { - match tokenizer { - None | Some(UserTurnTokenizer::Heuristic) => Ok(HeuristicCounter), - Some(UserTurnTokenizer::Cl100k) => Err(std::io::Error::other( - "cl100k tokenizer is not yet available in the Rust port; \ - omit `tokenizer` or pass `Some(Heuristic)` (see AgentWorkforce/burn#246)", - )), - } -} - -fn session_meta_payload_id(payload: &Value) -> Option { +pub(in crate::reader::codex) fn session_meta_payload_id(payload: &Value) -> Option { let id = payload.get("id")?.as_str()?; if id.is_empty() { None @@ -1244,7 +341,7 @@ fn session_meta_payload_id(payload: &Value) -> Option { } } -fn collect_message_text(payload: &Value, role: &str) -> String { +pub(in crate::reader::codex) fn collect_message_text(payload: &Value, role: &str) -> String { let Some(content) = payload.get("content").and_then(|v| v.as_array()) else { return String::new(); }; @@ -1287,7 +384,7 @@ fn is_codex_boilerplate(text: &str) -> bool { false } -fn collect_reasoning_text(payload: &Value) -> String { +pub(in crate::reader::codex) fn collect_reasoning_text(payload: &Value) -> String { let mut parts: Vec = Vec::new(); if let Some(arr) = payload.get("summary").and_then(|v| v.as_array()) { for s in arr { @@ -1310,7 +407,7 @@ fn collect_reasoning_text(payload: &Value) -> String { parts.join("\n") } -fn append_text(existing: &str, next: &str) -> String { +pub(in crate::reader::codex) fn append_text(existing: &str, next: &str) -> String { if existing.is_empty() { next.to_string() } else { @@ -1318,17 +415,9 @@ fn append_text(existing: &str, next: &str) -> String { } } -fn join_nonempty(parts: &[&str], sep: &str) -> String { - let mut out: Vec<&str> = Vec::with_capacity(parts.len()); - for p in parts { - if !p.is_empty() { - out.push(p); - } - } - out.join(sep) -} - -fn safe_parse_json_object(s: &str) -> Option> { +pub(in crate::reader::codex) fn safe_parse_json_object( + s: &str, +) -> Option> { if s.is_empty() { return None; } @@ -1339,7 +428,7 @@ fn safe_parse_json_object(s: &str) -> Option> { } } -fn pick_function_call_target( +pub(in crate::reader::codex) fn pick_function_call_target( name: &str, args: Option<&serde_json::Map>, ) -> Option { @@ -1358,7 +447,7 @@ fn pick_function_call_target( } } -fn pick_custom_tool_target(name: &str, input: &str) -> Option { +pub(in crate::reader::codex) fn pick_custom_tool_target(name: &str, input: &str) -> Option { if name != "apply_patch" { return None; } @@ -1384,7 +473,7 @@ fn pick_custom_tool_target(name: &str, input: &str) -> Option { None } -fn pick_string_field(value: &Value, keys: &[&str]) -> Option { +pub(in crate::reader::codex) fn pick_string_field(value: &Value, keys: &[&str]) -> Option { let obj = value.as_object()?; for k in keys { if let Some(s) = obj.get(*k).and_then(|v| v.as_str()) { @@ -1396,7 +485,7 @@ fn pick_string_field(value: &Value, keys: &[&str]) -> Option { None } -fn extract_spawned_agent_id(output: &Value) -> Option { +pub(in crate::reader::codex) fn extract_spawned_agent_id(output: &Value) -> Option { let obj_owner; let obj = match output { Value::Object(_) => output, @@ -1413,17 +502,17 @@ fn extract_spawned_agent_id(output: &Value) -> Option { } #[derive(Default)] -struct Measured { - length: Option, - hash: Option, +pub(in crate::reader::codex) struct Measured { + pub(in crate::reader::codex) length: Option, + pub(in crate::reader::codex) hash: Option, /// Raw UTF-8 byte length of the materialized payload. Same value as /// `length` for Codex (the legacy `content_length` already counted /// bytes here, not chars), but tracked separately so the /// `ToolResultEventRecord` shape stays consistent across sources. - byte_length: Option, + pub(in crate::reader::codex) byte_length: Option, } -fn measure_tool_output(output: &Value) -> Measured { +pub(in crate::reader::codex) fn measure_tool_output(output: &Value) -> Measured { match output { Value::Null => Measured::default(), Value::String(s) => Measured { @@ -1442,7 +531,7 @@ fn measure_tool_output(output: &Value) -> Measured { } } -fn is_subagent_terminal_notification(t: &str) -> bool { +pub(in crate::reader::codex) fn is_subagent_terminal_notification(t: &str) -> bool { if !t.starts_with("subagent_") { return false; } @@ -1452,7 +541,7 @@ fn is_subagent_terminal_notification(t: &str) -> bool { || t.ends_with("_terminated") } -fn subagent_notification_status(payload: &Value) -> ToolResultStatus { +pub(in crate::reader::codex) fn subagent_notification_status(payload: &Value) -> ToolResultStatus { if let Some(b) = payload.get("success").and_then(|v| v.as_bool()) { return if b { ToolResultStatus::Completed @@ -1475,7 +564,11 @@ fn subagent_notification_status(payload: &Value) -> ToolResultStatus { ToolResultStatus::Completed } -fn build_root_relationship(session_id: &str, ts: &str, meta: &Value) -> SessionRelationshipRecord { +pub(in crate::reader::codex) fn build_root_relationship( + session_id: &str, + ts: &str, + meta: &Value, +) -> SessionRelationshipRecord { let mut row = SessionRelationshipRecord { v: 1, source: RelationshipSourceKind::Codex, @@ -1498,7 +591,7 @@ fn build_root_relationship(session_id: &str, ts: &str, meta: &Value) -> SessionR row } -fn build_session_meta_relationships( +pub(in crate::reader::codex) fn build_session_meta_relationships( session_id: &str, meta: &Value, fallback_ts: &str, @@ -1568,7 +661,7 @@ fn apply_codex_session_meta_provenance(row: &mut SessionRelationshipRecord, meta } } -fn codex_relationship_key(row: &SessionRelationshipRecord) -> String { +pub(in crate::reader::codex) fn codex_relationship_key(row: &SessionRelationshipRecord) -> String { let source = match row.source { RelationshipSourceKind::Codex => "codex", RelationshipSourceKind::ClaudeCode => "claude-code", @@ -1597,7 +690,7 @@ fn codex_relationship_key(row: &SessionRelationshipRecord) -> String { ) } -fn maybe_emit_spawn_relationship( +pub(in crate::reader::codex) fn maybe_emit_spawn_relationship( open_turn: &mut OpenTurn, session_id: &str, info: &SpawnCallInfo, @@ -1638,7 +731,7 @@ fn maybe_emit_spawn_relationship( } } -fn push_content( +pub(in crate::reader::codex) fn push_content( open_turn: &mut Option, pending: &mut Vec, record: ContentRecord, @@ -1650,7 +743,7 @@ fn push_content( } } -fn build_codex_user_turn_record( +pub(in crate::reader::codex) fn build_codex_user_turn_record( slot: &UserTurnSlot, session_id: &str, following_message_id: &str, @@ -1675,7 +768,7 @@ fn build_codex_user_turn_record( } } -fn build_codex_compaction_event( +pub(in crate::reader::codex) fn build_codex_compaction_event( session_id: &str, ts: &str, preceding: Option<&CodexLastCompletedTurn>, @@ -1726,5 +819,13 @@ fn clone_resume(r: Option<&CodexResumeState>) -> CodexResumeState { // scope discussion. pub mod span_tree; +// Incremental parse engine: the `CodexParseState` streaming state machine, +// its `CommittedSnapshot` shadow, and the `parse_codex_buffer` driver the +// public `parse_codex_session*` entry points wrap. Split out of this file; the +// helpers/types above feed it per line. +mod incremental; + +use self::incremental::parse_codex_buffer; + #[cfg(test)] mod tests; diff --git a/crates/relayburn-sdk/src/reader/codex/incremental.rs b/crates/relayburn-sdk/src/reader/codex/incremental.rs new file mode 100644 index 00000000..b99018de --- /dev/null +++ b/crates/relayburn-sdk/src/reader/codex/incremental.rs @@ -0,0 +1,1014 @@ +//! Codex incremental-parse engine. +//! +//! Mechanically split out of `reader/codex.rs` to shrink the oversized root +//! file. Holds the per-line streaming state machine (`CodexParseState`), its +//! parallel committed shadow (`CommittedSnapshot`), and the driver +//! (`parse_codex_buffer`) that the public `parse_codex_session*` entry points +//! wrap. Behavior is unchanged from the inline implementation; only module +//! placement, visibility, and imports differ. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::io::BufRead; + +use serde_json::Value; + +use crate::reader::classifier::{classify_activity, ClassificationInput}; +use crate::reader::git::ProjectResolver; +use crate::reader::hash::args_hash; +use crate::reader::types::{ + CompactionEvent, ContentKind, ContentRecord, ContentRole, ContentStoreMode, ContentToolResult, + ContentToolUse, SessionRelationshipRecord, SourceKind, ToolCall, ToolResultEventRecord, + ToolResultEventSource, ToolResultStatus, TurnRecord, UserTurnBlock, UserTurnBlockKind, + UserTurnRecord, +}; +use crate::reader::user_turn::{join_nonempty, HeuristicCounter}; + +use super::{ + append_text, build_codex_compaction_event, build_codex_user_turn_record, + build_root_relationship, build_session_meta_relationships, codex_relationship_key, + collect_message_text, collect_reasoning_text, extract_spawned_agent_id, finalize_turn, + is_subagent_terminal_notification, maybe_emit_spawn_relationship, measure_tool_output, + pick_custom_tool_target, pick_function_call_target, pick_string_field, push_content, + safe_parse_json_object, session_meta_payload_id, subagent_notification_status, + CodexLastCompletedTurn, CodexResumeState, CodexTurnContext, CumulativeUsage, FinalizedTurn, + OpenTurn, ParseCodexIncrementalOptions, ParseCodexIncrementalResult, SpawnCallInfo, + UserTurnSlot, +}; + +#[derive(Debug, Clone)] +struct Pending { + offset: u64, + record: T, +} + +/// Owns the mutable WORKING state of the Codex parse pass. The driver +/// (`parse_codex_buffer`) keeps the parallel `committed_*` shadow set and +/// snapshots fields off this struct at `task_complete` commit boundaries, so +/// a trailing partial/un-terminated line's mutations are discarded. The final +/// result is assembled from the committed snapshots, never from this working +/// state directly. +struct CodexParseState { + session_id: String, + session_cwd: Option, + turn_contexts: HashMap, + cumulative: CumulativeUsage, + open_turn: Option, + pending_user_text: String, + pending_content: Vec, + finalized: Vec, + user_turn_slot: UserTurnSlot, + user_turns: Vec, + root_session_emitted: bool, + seen_session_meta_keys: BTreeSet, + next_event_index: u64, + tool_result_counters: HashMap, + last_completed_turn: Option, + pending_tool_result_events: Vec>, + pending_relationships: Vec>, + pending_compactions: Vec>, +} + +impl CodexParseState { + /// Initialize working state from the resume option, mirroring the prior + /// inline `resume.map(...)` initializers verbatim. + fn new(resume: Option<&CodexResumeState>) -> Self { + Self { + session_id: resume.map(|r| r.session_id.clone()).unwrap_or_default(), + session_cwd: resume.and_then(|r| r.session_cwd.clone()), + turn_contexts: resume.map(|r| r.turn_contexts.clone()).unwrap_or_default(), + cumulative: resume.map(|r| r.cumulative.clone()).unwrap_or_default(), + open_turn: None, + pending_user_text: String::new(), + pending_content: Vec::new(), + finalized: Vec::new(), + user_turn_slot: resume + .and_then(|r| r.user_turn_slot.as_ref()) + .map(UserTurnSlot::from_persisted) + .unwrap_or_default(), + user_turns: Vec::new(), + root_session_emitted: resume.map(|r| r.root_session_emitted).unwrap_or(false), + seen_session_meta_keys: resume + .map(|r| r.session_meta_relationship_keys.iter().cloned().collect()) + .unwrap_or_default(), + next_event_index: resume.map(|r| r.next_event_index).unwrap_or(0), + tool_result_counters: resume + .map(|r| r.tool_result_counters.clone()) + .unwrap_or_default(), + last_completed_turn: resume.and_then(|r| r.last_completed_turn.clone()), + pending_tool_result_events: Vec::new(), + pending_relationships: Vec::new(), + pending_compactions: Vec::new(), + } + } + + fn handle_session_meta(&mut self, payload: &Value, rec_timestamp: &str, line_end_offset: u64) { + if let Some(id) = session_meta_payload_id(payload) { + self.session_id = id; + } + if let Some(cwd) = payload.get("cwd").and_then(|v| v.as_str()) { + self.session_cwd = Some(cwd.to_string()); + if let Some(open) = self.open_turn.as_mut() { + if open.project.is_none() { + open.project = Some(cwd.to_string()); + } + } + } + if !self.session_id.is_empty() && !self.root_session_emitted { + self.root_session_emitted = true; + let ts = payload + .get("timestamp") + .and_then(|v| v.as_str()) + .unwrap_or(rec_timestamp); + self.pending_relationships.push(Pending { + offset: line_end_offset, + record: build_root_relationship(&self.session_id, ts, payload), + }); + } + if !self.session_id.is_empty() { + for row in build_session_meta_relationships(&self.session_id, payload, rec_timestamp) { + let key = codex_relationship_key(&row); + if self.seen_session_meta_keys.contains(&key) { + continue; + } + self.seen_session_meta_keys.insert(key); + self.pending_relationships.push(Pending { + offset: line_end_offset, + record: row, + }); + } + } + } + + fn handle_turn_context(&mut self, payload: &Value) { + let ctx = CodexTurnContext { + turn_id: payload + .get("turn_id") + .and_then(|v| v.as_str()) + .map(str::to_string), + cwd: payload + .get("cwd") + .and_then(|v| v.as_str()) + .map(str::to_string), + model: payload + .get("model") + .and_then(|v| v.as_str()) + .map(str::to_string), + }; + if let Some(tid) = ctx.turn_id.clone() { + self.turn_contexts.insert(tid.clone(), ctx.clone()); + if let Some(open) = self.open_turn.as_mut() { + if open.turn_id == tid { + if open.model.is_empty() { + if let Some(m) = ctx.model.as_deref() { + open.model = m.to_string(); + } + } + if open.project.is_none() { + if let Some(c) = ctx.cwd.as_deref() { + open.project = Some(c.to_string()); + } + } + } + } + } + } + + fn handle_compacted(&mut self, rec_timestamp: &str, line_end_offset: u64) { + if !self.session_id.is_empty() { + self.pending_compactions.push(Pending { + offset: line_end_offset, + record: build_codex_compaction_event( + &self.session_id, + rec_timestamp, + self.last_completed_turn.as_ref(), + ), + }); + } + } + + fn handle_event_msg( + &mut self, + payload: &Value, + rec_timestamp: &str, + capture_content: bool, + line_end_offset: u64, + committed: &mut CommittedSnapshot, + ) { + let pl_type = payload.get("type").and_then(|v| v.as_str()).unwrap_or(""); + match pl_type { + "token_count" => { + if let Some(total) = payload.get("info").and_then(|i| { + if i.is_null() { + None + } else { + i.get("total_token_usage") + } + }) { + let input_total = total + .get("input_tokens") + .and_then(|v| v.as_i64()) + .unwrap_or(0); + let cached = total + .get("cached_input_tokens") + .and_then(|v| v.as_i64()) + .unwrap_or(0); + self.cumulative.input = input_total - cached; + self.cumulative.cache_read = cached; + self.cumulative.output = total + .get("output_tokens") + .and_then(|v| v.as_i64()) + .unwrap_or(0); + self.cumulative.reasoning = total + .get("reasoning_output_tokens") + .and_then(|v| v.as_i64()) + .unwrap_or(0); + if let Some(open) = self.open_turn.as_mut() { + open.usage_observed = true; + } + } + } + "task_started" => { + let ts = rec_timestamp; + let turn_id = match payload.get("turn_id").and_then(|v| v.as_str()) { + Some(t) => t.to_string(), + None => return, + }; + if let Some(open) = self.open_turn.take() { + self.finalized.push(finalize_turn(open, &self.cumulative)); + } + if !self.user_turn_slot.blocks.is_empty() { + self.user_turns.push(build_codex_user_turn_record( + &self.user_turn_slot, + &self.session_id, + &turn_id, + ts, + )); + } + self.user_turn_slot = UserTurnSlot::default(); + let ctx = self.turn_contexts.get(&turn_id).cloned(); + let project = ctx + .as_ref() + .and_then(|c| c.cwd.clone()) + .or_else(|| self.session_cwd.clone()); + let mut open = OpenTurn { + turn_id: turn_id.clone(), + ts: ts.to_string(), + model: ctx + .as_ref() + .and_then(|c| c.model.clone()) + .unwrap_or_default(), + project, + start_cumulative: self.cumulative.clone(), + tool_calls: vec![], + seen_call_ids: BTreeSet::new(), + files_touched: BTreeSet::new(), + user_text: std::mem::take(&mut self.pending_user_text), + assistant_text: String::new(), + errored_call_ids: BTreeSet::new(), + content: vec![], + pending_tool_result_events: vec![], + pending_relationships: vec![], + spawn_calls: HashMap::new(), + usage_observed: false, + }; + if capture_content && !self.pending_content.is_empty() { + for c in self.pending_content.iter_mut() { + c.message_id = turn_id.clone(); + } + open.content.append(&mut self.pending_content); + } + self.open_turn = Some(open); + } + "task_complete" => { + let payload_turn_id = payload + .get("turn_id") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let mut took: Option = None; + if let Some(open) = self.open_turn.as_ref() { + if open.turn_id == payload_turn_id { + took = self.open_turn.take(); + } + } + if let Some(mut open) = took { + // Patch isError on tool-result blocks accumulated this turn + for b in self.user_turn_slot.blocks.iter_mut() { + if matches!(b.kind, UserTurnBlockKind::ToolResult) { + if let Some(id) = &b.tool_use_id { + if open.errored_call_ids.contains(id) { + b.is_error = Some(true); + } + } + } + } + // Drain pending tool result events / relationships + let mut events = std::mem::take(&mut open.pending_tool_result_events); + for ev in events.iter_mut() { + if open.errored_call_ids.contains(&ev.tool_use_id) { + ev.status = ToolResultStatus::Errored; + ev.is_error = Some(true); + } else if matches!(ev.status, ToolResultStatus::Unknown) { + ev.status = ToolResultStatus::Completed; + } + } + for ev in events { + self.pending_tool_result_events.push(Pending { + offset: line_end_offset, + record: ev, + }); + } + let rels = std::mem::take(&mut open.pending_relationships); + for r in rels { + self.pending_relationships.push(Pending { + offset: line_end_offset, + record: r, + }); + } + self.user_turn_slot.preceding_message_id = Some(open.turn_id.clone()); + let closed = finalize_turn(open, &self.cumulative); + self.last_completed_turn = Some(CodexLastCompletedTurn { + message_id: closed.turn_id.clone(), + cache_read: closed.usage.cache_read, + }); + self.finalized.push(closed); + // Commit snapshot + committed.snapshot(self, line_end_offset); + } + } + "patch_apply_end" => { + if let Some(open) = self.open_turn.as_mut() { + let turn_id = payload.get("turn_id").and_then(|v| v.as_str()); + if turn_id != Some(open.turn_id.as_str()) { + return; + } + let success = payload.get("success").and_then(|v| v.as_bool()); + if success == Some(false) { + if let Some(call_id) = payload.get("call_id").and_then(|v| v.as_str()) { + open.errored_call_ids.insert(call_id.to_string()); + } + return; + } + if let Some(changes) = payload.get("changes").and_then(|v| v.as_object()) { + for file in changes.keys() { + open.files_touched.insert(file.clone()); + } + } + } + } + "exec_command_end" => { + if let Some(open) = self.open_turn.as_mut() { + let turn_id = payload.get("turn_id").and_then(|v| v.as_str()); + if turn_id != Some(open.turn_id.as_str()) { + return; + } + let exit_code = payload.get("exit_code").and_then(|v| v.as_i64()); + if let (Some(code), Some(call_id)) = + (exit_code, payload.get("call_id").and_then(|v| v.as_str())) + { + if code != 0 { + open.errored_call_ids.insert(call_id.to_string()); + } + } + } + } + other if is_subagent_terminal_notification(other) => { + let call_id = match payload.get("call_id").and_then(|v| v.as_str()) { + Some(c) if !c.is_empty() => c.to_string(), + _ => return, + }; + let entry = self + .tool_result_counters + .entry(call_id.clone()) + .or_insert(0); + let call_index = *entry; + *entry += 1; + let status = subagent_notification_status(payload); + let mut ev = ToolResultEventRecord { + v: 1, + source: SourceKind::Codex, + session_id: self.session_id.clone(), + message_id: self.open_turn.as_ref().map(|o| o.turn_id.clone()), + tool_use_id: call_id.clone(), + call_index: Some(call_index), + event_index: self.next_event_index, + ts: if rec_timestamp.is_empty() { + None + } else { + Some(rec_timestamp.to_string()) + }, + status, + event_source: ToolResultEventSource::SubagentNotification, + content_length: None, + output_bytes: None, + output_truncated: None, + content_hash: None, + is_error: matches!(status, ToolResultStatus::Errored).then_some(true), + usage: None, + usage_attribution: None, + subagent_session_id: None, + agent_id: None, + replaced_tools: None, + collapsed_calls: None, + }; + self.next_event_index += 1; + let spawned_id = + pick_string_field(payload, &["agent_id", "subagent_id", "session_id"]); + if let Some(sid) = spawned_id.as_ref() { + ev.agent_id = Some(sid.clone()); + ev.subagent_session_id = Some(sid.clone()); + if let Some(open) = self.open_turn.as_mut() { + if let Some(spawn) = open.spawn_calls.get_mut(&call_id) { + if spawn.spawned_agent_id.is_none() { + spawn.spawned_agent_id = Some(sid.clone()); + let info = spawn.clone(); + maybe_emit_spawn_relationship( + open, + &self.session_id, + &info, + rec_timestamp, + ); + } + } + } + } + if let Some(open) = self.open_turn.as_mut() { + open.pending_tool_result_events.push(ev); + } else { + self.pending_tool_result_events.push(Pending { + offset: line_end_offset, + record: ev, + }); + } + } + _ => {} + } + } + + fn handle_response_item( + &mut self, + payload: &Value, + rec_timestamp: &str, + capture_content: bool, + line_end_offset: u64, + counter: &HeuristicCounter, + ) { + let item_ts = rec_timestamp; + let pl_type = payload.get("type").and_then(|v| v.as_str()).unwrap_or(""); + match pl_type { + "message" => { + let role = payload + .get("role") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let text = collect_message_text(payload, &role); + if text.is_empty() { + return; + } + if role == "user" { + if let Some(open) = self.open_turn.as_mut() { + open.user_text = append_text(&open.user_text, &text); + } else { + self.pending_user_text = append_text(&self.pending_user_text, &text); + } + self.user_turn_slot + .blocks + .push(UserTurnBlock::text(&text, counter)); + if self.user_turn_slot.ts.is_empty() && !item_ts.is_empty() { + self.user_turn_slot.ts = item_ts.to_string(); + } + if capture_content { + let rec = ContentRecord { + v: 1, + source: SourceKind::Codex, + session_id: self.session_id.clone(), + message_id: self + .open_turn + .as_ref() + .map(|o| o.turn_id.clone()) + .unwrap_or_default(), + ts: item_ts.to_string(), + role: ContentRole::User, + kind: ContentKind::Text, + text: Some(text.clone()), + tool_use: None, + tool_result: None, + }; + push_content(&mut self.open_turn, &mut self.pending_content, rec); + } + } else if role == "assistant" { + if let Some(open) = self.open_turn.as_mut() { + open.assistant_text = append_text(&open.assistant_text, &text); + if capture_content { + open.content.push(ContentRecord { + v: 1, + source: SourceKind::Codex, + session_id: self.session_id.clone(), + message_id: open.turn_id.clone(), + ts: item_ts.to_string(), + role: ContentRole::Assistant, + kind: ContentKind::Text, + text: Some(text), + tool_use: None, + tool_result: None, + }); + } + } + } + } + "reasoning" => { + if !capture_content { + return; + } + let Some(open) = self.open_turn.as_mut() else { + return; + }; + let text = collect_reasoning_text(payload); + if !text.is_empty() { + open.content.push(ContentRecord { + v: 1, + source: SourceKind::Codex, + session_id: self.session_id.clone(), + message_id: open.turn_id.clone(), + ts: item_ts.to_string(), + role: ContentRole::Assistant, + kind: ContentKind::Thinking, + text: Some(text), + tool_use: None, + tool_result: None, + }); + } + } + "function_call_output" | "custom_tool_call_output" => { + let call_id = match payload.get("call_id").and_then(|v| v.as_str()) { + Some(c) => c.to_string(), + None => return, + }; + let output = payload.get("output").cloned().unwrap_or(Value::Null); + self.user_turn_slot.blocks.push(UserTurnBlock::tool_result( + call_id.clone(), + &output, + None, + counter, + )); + if self.user_turn_slot.ts.is_empty() && !item_ts.is_empty() { + self.user_turn_slot.ts = item_ts.to_string(); + } + let entry = self + .tool_result_counters + .entry(call_id.clone()) + .or_insert(0); + let call_index = *entry; + *entry += 1; + let initial_status = if self + .open_turn + .as_ref() + .map(|o| o.errored_call_ids.contains(&call_id)) + .unwrap_or(false) + { + ToolResultStatus::Errored + } else { + ToolResultStatus::Unknown + }; + let measured = measure_tool_output(&output); + let mut ev = ToolResultEventRecord { + v: 1, + source: SourceKind::Codex, + session_id: self.session_id.clone(), + message_id: self.open_turn.as_ref().map(|o| o.turn_id.clone()), + tool_use_id: call_id.clone(), + call_index: Some(call_index), + event_index: self.next_event_index, + ts: if item_ts.is_empty() { + None + } else { + Some(item_ts.to_string()) + }, + status: initial_status, + event_source: ToolResultEventSource::FunctionCallOutput, + content_length: measured.length, + output_bytes: measured.byte_length, + // Codex doesn't carry an explicit truncation marker + // distinct from its general output; leave None until + // we have a concrete signal to flip on. + output_truncated: None, + content_hash: measured.hash, + is_error: matches!(initial_status, ToolResultStatus::Errored).then_some(true), + usage: None, + usage_attribution: None, + subagent_session_id: None, + agent_id: None, + replaced_tools: None, + collapsed_calls: None, + }; + self.next_event_index += 1; + if let Some(open) = self.open_turn.as_mut() { + if let Some(spawn) = open.spawn_calls.get_mut(&call_id) { + if let Some(sid) = extract_spawned_agent_id(&output) { + spawn.spawned_agent_id = Some(sid.clone()); + ev.agent_id = Some(sid.clone()); + ev.subagent_session_id = Some(sid); + } + let info = spawn.clone(); + maybe_emit_spawn_relationship(open, &self.session_id, &info, item_ts); + } + open.pending_tool_result_events.push(ev); + } else { + self.pending_tool_result_events.push(Pending { + offset: line_end_offset, + record: ev, + }); + } + if capture_content { + let rec = ContentRecord { + v: 1, + source: SourceKind::Codex, + session_id: self.session_id.clone(), + message_id: self + .open_turn + .as_ref() + .map(|o| o.turn_id.clone()) + .unwrap_or_default(), + ts: item_ts.to_string(), + role: ContentRole::ToolResult, + kind: ContentKind::ToolResult, + text: None, + tool_use: None, + tool_result: Some(ContentToolResult { + tool_use_id: call_id, + content: output, + is_error: None, + }), + }; + push_content(&mut self.open_turn, &mut self.pending_content, rec); + } + } + "function_call" => { + let Some(open) = self.open_turn.as_mut() else { + return; + }; + let name = match payload.get("name").and_then(|v| v.as_str()) { + Some(n) => n.to_string(), + None => return, + }; + let call_id = match payload.get("call_id").and_then(|v| v.as_str()) { + Some(c) => c.to_string(), + None => return, + }; + if open.seen_call_ids.contains(&call_id) { + return; + } + open.seen_call_ids.insert(call_id.clone()); + let arg_str = payload.get("arguments").and_then(|v| v.as_str()); + let parsed_args = arg_str.and_then(safe_parse_json_object); + let hash_input = parsed_args + .clone() + .map(Value::Object) + .unwrap_or_else(|| Value::Object(Default::default())); + let target = pick_function_call_target(&name, parsed_args.as_ref()); + let call = ToolCall { + id: call_id.clone(), + name: name.clone(), + target, + args_hash: args_hash(&hash_input), + is_error: None, + edit_pre_hash: None, + edit_post_hash: None, + skill_name: None, + replaced_tools: None, + collapsed_calls: None, + }; + open.tool_calls.push(call); + if name == "spawn_agent" { + let mut info = SpawnCallInfo { + call_id: call_id.clone(), + ts: item_ts.to_string(), + subagent_type: None, + description: None, + spawned_agent_id: None, + emitted: false, + }; + if let Some(args) = parsed_args.as_ref() { + let v = Value::Object(args.clone()); + info.subagent_type = + pick_string_field(&v, &["subagent_type", "agent_type", "type"]); + info.description = + pick_string_field(&v, &["description", "task", "prompt"]); + info.spawned_agent_id = + pick_string_field(&v, &["agent_id", "subagent_id", "session_id"]); + } + open.spawn_calls.insert(call_id.clone(), info.clone()); + maybe_emit_spawn_relationship(open, &self.session_id, &info, item_ts); + } + if capture_content { + let input = parsed_args + .map(|m| m.into_iter().collect()) + .unwrap_or_default(); + open.content.push(ContentRecord { + v: 1, + source: SourceKind::Codex, + session_id: self.session_id.clone(), + message_id: open.turn_id.clone(), + ts: item_ts.to_string(), + role: ContentRole::Assistant, + kind: ContentKind::ToolUse, + text: None, + tool_use: Some(ContentToolUse { + id: call_id, + name, + input, + }), + tool_result: None, + }); + } + } + "custom_tool_call" => { + let Some(open) = self.open_turn.as_mut() else { + return; + }; + let name = match payload.get("name").and_then(|v| v.as_str()) { + Some(n) => n.to_string(), + None => return, + }; + let call_id = match payload.get("call_id").and_then(|v| v.as_str()) { + Some(c) => c.to_string(), + None => return, + }; + if open.seen_call_ids.contains(&call_id) { + return; + } + open.seen_call_ids.insert(call_id.clone()); + let input = payload + .get("input") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let hash_input = serde_json::json!({ "input": input }); + let target = pick_custom_tool_target(&name, &input); + let call = ToolCall { + id: call_id.clone(), + name: name.clone(), + target, + args_hash: args_hash(&hash_input), + is_error: None, + edit_pre_hash: None, + edit_post_hash: None, + skill_name: None, + replaced_tools: None, + collapsed_calls: None, + }; + open.tool_calls.push(call); + if capture_content { + let mut input_map = BTreeMap::new(); + input_map.insert("input".to_string(), Value::String(input)); + open.content.push(ContentRecord { + v: 1, + source: SourceKind::Codex, + session_id: self.session_id.clone(), + message_id: open.turn_id.clone(), + ts: item_ts.to_string(), + role: ContentRole::Assistant, + kind: ContentKind::ToolUse, + text: None, + tool_use: Some(ContentToolUse { + id: call_id, + name, + input: input_map, + }), + tool_result: None, + }); + } + } + _ => {} + } + } +} + +/// The parallel `committed_*` shadow set. Snapshots advance only at +/// `task_complete` boundaries; the final result is assembled from these +/// fields so a trailing partial line's working-state mutations are dropped. +struct CommittedSnapshot { + end_offset: u64, + cumulative: CumulativeUsage, + session_id: String, + session_cwd: Option, + turn_contexts: HashMap, + finalized_count: usize, + user_turns_count: usize, + user_turn_slot: UserTurnSlot, + root_session_emitted: bool, + seen_session_meta_keys: BTreeSet, + next_event_index: u64, + tool_result_counters: HashMap, + last_completed_turn: Option, +} + +impl CommittedSnapshot { + fn initial(state: &CodexParseState, start_offset: u64) -> Self { + Self { + end_offset: start_offset, + cumulative: state.cumulative.clone(), + session_id: state.session_id.clone(), + session_cwd: state.session_cwd.clone(), + turn_contexts: state.turn_contexts.clone(), + finalized_count: 0, + user_turns_count: 0, + user_turn_slot: state.user_turn_slot.clone(), + root_session_emitted: state.root_session_emitted, + seen_session_meta_keys: state.seen_session_meta_keys.clone(), + next_event_index: state.next_event_index, + tool_result_counters: state.tool_result_counters.clone(), + last_completed_turn: state.last_completed_turn.clone(), + } + } + + /// Advance the committed snapshot to the current working state at a + /// `task_complete` commit boundary. Mirrors the prior inline assignments + /// verbatim. + fn snapshot(&mut self, state: &CodexParseState, line_end_offset: u64) { + self.end_offset = line_end_offset; + self.cumulative = state.cumulative.clone(); + self.session_id = state.session_id.clone(); + self.session_cwd = state.session_cwd.clone(); + self.turn_contexts = state.turn_contexts.clone(); + self.finalized_count = state.finalized.len(); + self.user_turns_count = state.user_turns.len(); + self.user_turn_slot = state.user_turn_slot.clone(); + self.root_session_emitted = state.root_session_emitted; + self.seen_session_meta_keys = state.seen_session_meta_keys.clone(); + self.next_event_index = state.next_event_index; + self.tool_result_counters = state.tool_result_counters.clone(); + self.last_completed_turn = state.last_completed_turn.clone(); + } +} + +pub(super) fn parse_codex_buffer( + mut reader: R, + start_offset: u64, + options: &ParseCodexIncrementalOptions, + project_resolver: &ProjectResolver, +) -> std::io::Result { + let capture_content = matches!(options.content_mode, Some(ContentStoreMode::Full)); + // Validated by `resolve_token_counter` at the public entry point. + let counter = HeuristicCounter; + + let mut state = CodexParseState::new(options.resume.as_ref()); + let mut committed = CommittedSnapshot::initial(&state, start_offset); + + let mut line_buf: Vec = Vec::new(); + let mut current_offset: u64 = start_offset; + loop { + line_buf.clear(); + let n = reader.read_until(b'\n', &mut line_buf)?; + if n == 0 { + break; + } + // Drop trailing partial lines — the next incremental call resumes + // from the committed end offset, which only advances past `\n`. + if line_buf.last() != Some(&b'\n') { + break; + } + let line_end_offset = current_offset + n as u64; + current_offset = line_end_offset; + let text = std::str::from_utf8(&line_buf[..n - 1]).unwrap_or("").trim(); + if text.is_empty() { + continue; + } + let parsed: Value = match serde_json::from_str(text) { + Ok(v) => v, + Err(_) => continue, + }; + if !parsed.is_object() { + continue; + } + let rec_type = parsed.get("type").and_then(|v| v.as_str()).unwrap_or(""); + let rec_timestamp = parsed + .get("timestamp") + .and_then(|v| v.as_str()) + .unwrap_or(""); + let payload = match parsed.get("payload") { + Some(p) if p.is_object() => p, + _ => continue, + }; + + match rec_type { + "session_meta" => state.handle_session_meta(payload, rec_timestamp, line_end_offset), + "turn_context" => state.handle_turn_context(payload), + "compacted" => state.handle_compacted(rec_timestamp, line_end_offset), + "event_msg" => state.handle_event_msg( + payload, + rec_timestamp, + capture_content, + line_end_offset, + &mut committed, + ), + "response_item" => state.handle_response_item( + payload, + rec_timestamp, + capture_content, + line_end_offset, + &counter, + ), + _ => continue, + } + } + + // Emit only committed turns. + let committed_turns = &state.finalized[..committed.finalized_count]; + let mut turns: Vec = Vec::with_capacity(committed_turns.len()); + let mut content_out: Vec = Vec::new(); + for (i, f) in committed_turns.iter().enumerate() { + let mut record = TurnRecord { + v: 1, + source: SourceKind::Codex, + session_id: committed.session_id.clone(), + session_path: options.session_path.clone(), + message_id: f.turn_id.clone(), + turn_index: i as u64, + ts: f.ts.clone(), + model: f.model.clone(), + project: None, + project_key: None, + usage: f.usage.clone(), + tool_calls: f.tool_calls.clone(), + files_touched: if f.files_touched.is_empty() { + None + } else { + Some(f.files_touched.clone()) + }, + subagent: None, + stop_reason: None, + activity: None, + retries: None, + has_edits: None, + fidelity: Some(f.fidelity.clone()), + }; + if let Some(p) = f.project.as_ref() { + let resolved = project_resolver.resolve(p); + record.project = Some(resolved.project); + record.project_key = resolved.project_key; + } + let combined_text = join_nonempty(&[f.user_text.as_str(), f.assistant_text.as_str()], "\n"); + let has_failed_tool = f + .tool_calls + .iter() + .any(|tc| f.errored_call_ids.contains(&tc.id)); + let classified = classify_activity(ClassificationInput { + tool_calls: &f.tool_calls, + text: &combined_text, + has_failed_tool, + reasoning_tokens: f.usage.reasoning, + }); + record.activity = Some(classified.activity); + record.retries = Some(classified.retries); + record.has_edits = Some(classified.has_edits); + turns.push(record); + if capture_content { + content_out.extend(f.content.clone()); + } + } + + let resume = CodexResumeState { + cumulative: committed.cumulative.clone(), + session_id: committed.session_id.clone(), + session_cwd: committed.session_cwd.clone(), + turn_contexts: committed.turn_contexts.clone(), + user_turn_slot: Some(committed.user_turn_slot.to_persisted()), + root_session_emitted: committed.root_session_emitted, + session_meta_relationship_keys: committed.seen_session_meta_keys.iter().cloned().collect(), + next_event_index: committed.next_event_index, + tool_result_counters: committed.tool_result_counters.clone(), + last_completed_turn: committed.last_completed_turn.clone(), + }; + + let user_turns_out = state.user_turns[..committed.user_turns_count].to_vec(); + let mut events_out: Vec = Vec::new(); + for e in state.pending_compactions { + if e.offset <= committed.end_offset { + events_out.push(e.record); + } + } + let mut relationships_out: Vec = Vec::new(); + for r in state.pending_relationships { + if r.offset <= committed.end_offset { + relationships_out.push(r.record); + } + } + let mut tool_events_out: Vec = Vec::new(); + for ev in state.pending_tool_result_events { + if ev.offset <= committed.end_offset { + tool_events_out.push(ev.record); + } + } + + Ok(ParseCodexIncrementalResult { + turns, + content: content_out, + events: events_out, + user_turns: user_turns_out, + relationships: relationships_out, + tool_result_events: tool_events_out, + end_offset: committed.end_offset, + resume, + }) +} diff --git a/crates/relayburn-sdk/src/reader/opencode.rs b/crates/relayburn-sdk/src/reader/opencode.rs index 1c83792c..c4299e88 100644 --- a/crates/relayburn-sdk/src/reader/opencode.rs +++ b/crates/relayburn-sdk/src/reader/opencode.rs @@ -36,7 +36,9 @@ use crate::reader::types::{ ToolResultEventSource, ToolResultStatus, TurnRecord, Usage, UsageAttribution, UsageGranularity, UserTurnBlock, UserTurnRecord, }; -use crate::reader::user_turn::{HeuristicCounter, TokenCounter, UserTurnTokenizer}; +use crate::reader::user_turn::{ + join_nonempty, resolve_token_counter, HeuristicCounter, TokenCounter, UserTurnTokenizer, +}; // --------------------------------------------------------------------------- // Public surface @@ -1272,27 +1274,5 @@ fn find_preceding_assistant_by_time( best } -fn join_nonempty(parts: &[&str], sep: &str) -> String { - let mut out: Vec<&str> = Vec::with_capacity(parts.len()); - for p in parts { - if !p.is_empty() { - out.push(p); - } - } - out.join(sep) -} - -fn resolve_token_counter( - tokenizer: Option, -) -> std::io::Result { - match tokenizer { - None | Some(UserTurnTokenizer::Heuristic) => Ok(HeuristicCounter), - Some(UserTurnTokenizer::Cl100k) => Err(std::io::Error::other( - "cl100k tokenizer is not yet available in the Rust port; \ - omit `tokenizer` or pass `Some(Heuristic)` (see AgentWorkforce/burn#246)", - )), - } -} - #[cfg(test)] mod tests; diff --git a/crates/relayburn-sdk/src/reader/types.rs b/crates/relayburn-sdk/src/reader/types.rs index 14ceee50..4387b4b7 100644 --- a/crates/relayburn-sdk/src/reader/types.rs +++ b/crates/relayburn-sdk/src/reader/types.rs @@ -157,7 +157,7 @@ pub struct Subagent { pub description: Option, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum UsageGranularity { PerTurn, @@ -237,7 +237,7 @@ impl Default for Coverage { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum FidelityClass { Full, diff --git a/crates/relayburn-sdk/src/reader/user_turn.rs b/crates/relayburn-sdk/src/reader/user_turn.rs index ce8e7537..23c9c115 100644 --- a/crates/relayburn-sdk/src/reader/user_turn.rs +++ b/crates/relayburn-sdk/src/reader/user_turn.rs @@ -118,6 +118,38 @@ pub fn bytes_to_approx_tokens(byte_len: u64) -> u64 { } } +/// Resolves the requested tokenizer to a concrete counter. `None` and +/// `Some(Heuristic)` map to [`HeuristicCounter`]; `Some(Cl100k)` is rejected +/// with an explicit error until the cl100k counter is wired up (see #246) so +/// callers don't silently get bytes/4 sizing when they asked for cl100k. +/// +/// Shared by the codex and opencode readers, which validate the requested +/// tokenizer at their public entry points. +pub(crate) fn resolve_token_counter( + tokenizer: Option, +) -> std::io::Result { + match tokenizer { + None | Some(UserTurnTokenizer::Heuristic) => Ok(HeuristicCounter), + Some(UserTurnTokenizer::Cl100k) => Err(std::io::Error::other( + "cl100k tokenizer is not yet available in the Rust port; \ + omit `tokenizer` or pass `Some(Heuristic)` (see AgentWorkforce/burn#246)", + )), + } +} + +/// Join the non-empty entries of `parts` with `sep`, skipping empties so an +/// absent half (e.g. a user turn with no assistant text) doesn't leave a +/// dangling separator. Used by the readers to assemble combined user-turn text. +pub(crate) fn join_nonempty(parts: &[&str], sep: &str) -> String { + let mut out: Vec<&str> = Vec::with_capacity(parts.len()); + for p in parts { + if !p.is_empty() { + out.push(p); + } + } + out.join(sep) +} + #[cfg(test)] mod tests { use super::*; diff --git a/packages/mcp/CHANGELOG.md b/packages/mcp/CHANGELOG.md index 41f88fe9..ab18ef96 100644 --- a/packages/mcp/CHANGELOG.md +++ b/packages/mcp/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to `@relayburn/mcp`. ## [Unreleased] +- `compare` / `summary` tool output reflects the SDK's canonical compare-cost rounding (`toFixed` semantics; ties shift by one in the last digit) and a stable, deterministic fidelity-summary key order. + ## [3.0.0] - 2026-05-26 ### Added diff --git a/packages/relayburn/CHANGELOG.md b/packages/relayburn/CHANGELOG.md index 82653f57..3fe8667d 100644 --- a/packages/relayburn/CHANGELOG.md +++ b/packages/relayburn/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to `relayburn`. ## [Unreleased] +- `burn compare` cost figures now use canonical decimal rounding (`{:.N}`/`toFixed` semantics), so cells/totals/buckets can shift by one in the last reported digit at exact ties. +- `fidelity` blocks in `summary` / `compare` JSON now emit `byClass` / `byGranularity` / `missingCoverage` keys in a stable order, so output is reproducible across runs. + ## [3.4.0] - 2026-06-20 - Added `--bucket ` to `burn summary` and `burn compare` for a per-bucket time-series across the `--since` window (`{ bucketSeconds, buckets: [...] }` in `--json`). Grammar: `30s`/`5m`(minutes)/`1h`/`12h`/`1d`/`7d`. diff --git a/packages/sdk-node/CHANGELOG.md b/packages/sdk-node/CHANGELOG.md index 5d303d29..e7b4f618 100644 --- a/packages/sdk-node/CHANGELOG.md +++ b/packages/sdk-node/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] +- `compare()` cost figures now use canonical decimal rounding (`toFixed` semantics) instead of float-multiply rounding, so cells/totals can shift by one in the last reported digit at exact ties. +- Fidelity summaries (`byClass` / `byGranularity` / `missingCoverage`) now have a stable key order instead of a randomized per-call order, so results are reproducible. + ## [3.2.2] - 2026-06-10 ### Changed