diff --git a/CHANGELOG.md b/CHANGELOG.md index d0eaa5f8..be2c88bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Cross-package release notes for relayburn. Package changelogs contain package-le ## [Unreleased] +- `burn summary` now reports turns whose model has no pricing entry (`unpricedTurns`/`unpricedModels` in JSON output, warning footer in human output) instead of silently counting them at $0. + ## [3.2.1] - 2026-06-09 ### Added diff --git a/crates/relayburn-cli/src/commands/summary.rs b/crates/relayburn-cli/src/commands/summary.rs index 94b66392..bf62ddd3 100644 --- a/crates/relayburn-cli/src/commands/summary.rs +++ b/crates/relayburn-cli/src/commands/summary.rs @@ -1014,6 +1014,17 @@ fn emit_human(report: &SummaryGroupedReport, ingest_report: &relayburn_sdk::Inge let out = lines.join("\n"); // TS uses `process.stdout.write(lines.join('\n'))` — no trailing newline. print!("{}", out); + + if report.unpriced_turns > 0 { + let models = report.unpriced_models.join(", "); + eprintln!( + "warning: {} turn(s) had no pricing for model(s): {} — their cost is reported as $0.", + report.unpriced_turns, models, + ); + eprintln!( + " Update the snapshot (pnpm run pricing:update) or add an override at $RELAYBURN_HOME/models.dev.json.", + ); + } } fn render_quality(q: &QualityResult) -> String { @@ -1271,6 +1282,8 @@ mod tests { stop_reasons: relayburn_sdk::StopReasonCounts::default(), subagents: SubagentCounts::default(), quality: Some(QualityResult::default()), + unpriced_turns: 0, + unpriced_models: Vec::new(), }; let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty()); @@ -1324,6 +1337,8 @@ mod tests { stop_reasons: relayburn_sdk::StopReasonCounts::default(), subagents: SubagentCounts::default(), quality: None, + unpriced_turns: 0, + unpriced_models: Vec::new(), }; let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty()); assert!( diff --git a/crates/relayburn-sdk/src/analyze.rs b/crates/relayburn-sdk/src/analyze.rs index 8424d474..e4dce047 100644 --- a/crates/relayburn-sdk/src/analyze.rs +++ b/crates/relayburn-sdk/src/analyze.rs @@ -50,7 +50,7 @@ pub use context_delta::{ deltas_for_session, ContextDelta, ContextDeltaOpts, InterveningStep, OwnerFilter, OwnerRail, ReminderSource, }; -pub use cost::{cost_for_turn, cost_for_usage, sum_costs, CostBreakdown}; +pub use cost::{cost_for_turn, cost_for_usage, sum_costs, tally_unpriced, CostBreakdown}; pub use fidelity::{ has_minimum_fidelity, summarize_fidelity, summarize_fidelity_from_iter, FidelitySummary, }; diff --git a/crates/relayburn-sdk/src/analyze/cost.rs b/crates/relayburn-sdk/src/analyze/cost.rs index 243a3781..e56b3c09 100644 --- a/crates/relayburn-sdk/src/analyze/cost.rs +++ b/crates/relayburn-sdk/src/analyze/cost.rs @@ -147,6 +147,23 @@ fn strip_provider_prefix(model: &str) -> &str { } } +/// Count turns whose model has no pricing entry, and collect the distinct +/// model names, first-seen order. Used by summary surfaces to make pricing +/// gaps visible instead of silently folding them in at $0. +pub fn tally_unpriced(turns: &[TurnRecord], pricing: &PricingTable) -> (u64, Vec) { + let mut count = 0u64; + let mut models: Vec = Vec::new(); + for t in turns { + if lookup_model_rate(&t.model, pricing).is_none() { + count += 1; + if !models.iter().any(|m| m == &t.model) { + models.push(t.model.clone()); + } + } + } + (count, models) +} + pub fn sum_costs(costs: I) -> CostBreakdown where I: IntoIterator, @@ -585,6 +602,56 @@ mod tests { assert_eq!(s.output, 1.5); } + #[test] + fn tally_unpriced_returns_zero_when_all_models_priced() { + let p = load_builtin_pricing(); + let turns = vec![ + turn( + "claude-sonnet-4-6", + usage_with(100, 50, 0), + SourceKind::ClaudeCode, + ), + turn( + "claude-opus-4-7", + usage_with(200, 100, 0), + SourceKind::ClaudeCode, + ), + ]; + let (count, models) = tally_unpriced(&turns, &p); + assert_eq!(count, 0); + assert!(models.is_empty()); + } + + #[test] + fn tally_unpriced_counts_turns_and_deduplicates_models() { + let p = load_builtin_pricing(); + // "made-up-model-xyz" is not in the pricing table; appears in two turns. + let turns = vec![ + turn( + "made-up-model-xyz", + usage_with(100, 50, 0), + SourceKind::ClaudeCode, + ), + turn( + "made-up-model-xyz", + usage_with(200, 100, 0), + SourceKind::ClaudeCode, + ), + turn( + "claude-sonnet-4-6", + usage_with(300, 150, 0), + SourceKind::ClaudeCode, + ), + ]; + let (count, models) = tally_unpriced(&turns, &p); + assert_eq!(count, 2, "two turns used the unknown model"); + assert_eq!( + models, + vec!["made-up-model-xyz"], + "model listed exactly once" + ); + } + #[test] fn sum_costs_on_empty_input_returns_zero_aggregate() { let empty: Vec = vec![]; diff --git a/crates/relayburn-sdk/src/query_verbs.rs b/crates/relayburn-sdk/src/query_verbs.rs index 32b49d41..0169dd4d 100644 --- a/crates/relayburn-sdk/src/query_verbs.rs +++ b/crates/relayburn-sdk/src/query_verbs.rs @@ -27,16 +27,17 @@ use crate::analyze::{ load_claude_settings, load_overhead_file, load_pricing, project_claude_settings_path, provider_for, render_unified_diff_for_recommendation, sort_findings, sum_costs, summarize_fidelity, summarize_fidelity_from_iter, summarize_replacement_savings, - tool_call_pattern_to_finding, tool_output_bloat_to_finding, user_claude_settings_path, - AggregateByProviderOptions, AttributeOverheadInput, AttributionMethod, BashAggregation, - BashVerbAggregation, BuildSubagentTreeOptions, CompareOptions as AnalyzeCompareOptions, - CompareTable, ComputeQualityOptions, CostBreakdown, CoverageField, DetectPatternsOptions, - DetectToolCallPatternsOptions, DetectToolOutputBloatOptions, FidelitySummary, FieldCoverage, - FileAggregation, GhostSurfaceFindingOptions, HotspotsOptions as AnalyzeHotspotsOptions, - LoadedClaudeSettings, MarkdownSection, McpServerAggregation, OverheadFile, OverheadFileKind, - ParsedOverheadFile, PricingTable, ProviderAggregateRow, ProviderFilter, QualityResult, - ReplacementSavingsSummary, RowCoverage, SessionClaudeMdCost, SubagentAggregation, - SubagentTreeNode, SubagentTypeStats, ToolSavingsAggregate, UsageCostAggregateRow, WasteFinding, + tally_unpriced, tool_call_pattern_to_finding, tool_output_bloat_to_finding, + user_claude_settings_path, AggregateByProviderOptions, AttributeOverheadInput, + AttributionMethod, BashAggregation, BashVerbAggregation, BuildSubagentTreeOptions, + CompareOptions as AnalyzeCompareOptions, CompareTable, ComputeQualityOptions, CostBreakdown, + CoverageField, DetectPatternsOptions, DetectToolCallPatternsOptions, + DetectToolOutputBloatOptions, FidelitySummary, FieldCoverage, FileAggregation, + GhostSurfaceFindingOptions, HotspotsOptions as AnalyzeHotspotsOptions, LoadedClaudeSettings, + MarkdownSection, McpServerAggregation, OverheadFile, OverheadFileKind, ParsedOverheadFile, + PricingTable, ProviderAggregateRow, ProviderFilter, QualityResult, ReplacementSavingsSummary, + RowCoverage, SessionClaudeMdCost, SubagentAggregation, SubagentTreeNode, SubagentTypeStats, + ToolSavingsAggregate, UsageCostAggregateRow, WasteFinding, }; use crate::ledger::{EnrichedTurn, Enrichment, Query}; use crate::reader::{ @@ -496,6 +497,14 @@ pub struct Summary { /// Counts roll up the trailing `stop_reason` of every assistant turn /// in the filtered slice. See #437. pub stop_reasons: StopReasonCounts, + /// Count of turns whose model had no entry in the pricing snapshot. + /// Their cost is reported as $0. Zero when all models are priced. + #[serde(default)] + pub unpriced_turns: u64, + /// Distinct model names (first-seen order) that had no pricing entry. + /// Empty when all models are priced. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub unpriced_models: Vec, } impl LedgerHandle { @@ -602,6 +611,10 @@ fn compute_summary(turns: &[TurnRecord], pricing: &PricingTable) -> Summary { None }; + // Use the same pricing table that was used for cost accumulation so the + // count precisely matches which turns contributed $0 to `total_cost`. + let (unpriced_turns, unpriced_models) = tally_unpriced(turns, pricing); + Summary { total_tokens, total_cost, @@ -617,6 +630,8 @@ fn compute_summary(turns: &[TurnRecord], pricing: &PricingTable) -> Summary { by_tag: None, replacement_savings, stop_reasons: StopReasonCounts::from_turns(turns), + unpriced_turns, + unpriced_models, } } @@ -792,6 +807,14 @@ pub struct SummaryGroupedReport { pub subagents: crate::reader::SubagentCounts, #[serde(default, skip_serializing_if = "Option::is_none")] pub quality: Option, + /// Count of turns whose model had no entry in the pricing snapshot. + /// Their cost is reported as $0. Zero when all models are priced. + #[serde(default)] + pub unpriced_turns: u64, + /// Distinct model names (first-seen order) that had no pricing entry. + /// Empty when all models are priced. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub unpriced_models: Vec, } #[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] @@ -982,6 +1005,7 @@ impl LedgerHandle { // behavior. let session_filter = summary_subagent_session_filter(&opts, &turns); let subagents = compute_summary_subagent_counts(session_filter.as_ref()); + let (unpriced_turns, unpriced_models) = tally_unpriced(&turns, &pricing); Ok(SummaryReport::Grouped(SummaryGroupedReport { group_by, tag_key, @@ -995,6 +1019,8 @@ impl LedgerHandle { stop_reasons, subagents, quality, + unpriced_turns, + unpriced_models, })) } SummaryReportMode::ByTool => { @@ -5046,6 +5072,145 @@ mod tests { assert!(!grouped.stop_reasons.is_empty()); } + /// `compute_summary` (the slim legacy verb) populates unpriced_turns + /// and unpriced_models when a turn's model is absent from the pricing table. + #[test] + fn compute_summary_tracks_unpriced_turns_and_models() { + let pricing = load_pricing(None); + let unknown_model = "made-up-model-xyz"; + let priced_turn = TurnRecord { + v: 1, + source: SourceKind::ClaudeCode, + session_id: "s".into(), + session_path: None, + message_id: "m-priced".into(), + turn_index: 0, + ts: "2026-05-01T00:00:00.000Z".into(), + model: "claude-sonnet-4-6".into(), + project: None, + project_key: None, + usage: Usage { + input: 100, + output: 50, + reasoning: 0, + cache_read: 0, + cache_create_5m: 0, + cache_create_1h: 0, + }, + tool_calls: vec![], + files_touched: None, + subagent: None, + stop_reason: None, + activity: None, + retries: None, + has_edits: None, + fidelity: None, + }; + let unpriced_turn = TurnRecord { + message_id: "m-unpriced".into(), + turn_index: 1, + ts: "2026-05-01T00:01:00.000Z".into(), + model: unknown_model.into(), + ..priced_turn.clone() + }; + let turns = vec![priced_turn, unpriced_turn]; + let s = compute_summary(&turns, &pricing); + assert_eq!(s.turn_count, 2); + assert_eq!(s.unpriced_turns, 1, "one turn uses an unknown model"); + assert_eq!( + s.unpriced_models, + vec![unknown_model], + "unknown model listed exactly once" + ); + } + + /// `summary_report` (grouped mode) surfaces unpriced turn count and model + /// names when a turn's model is absent from the pricing snapshot. + #[test] + fn summary_report_grouped_tracks_unpriced_turns_and_models() { + let dir = tempfile::tempdir().unwrap(); + let opts = LedgerOpenOptions::with_home(dir.path()); + let mut handle = Ledger::open(opts).expect("open ledger"); + + let make_turn = |idx: u64, msg: &str, model: &str, ts: &str| -> TurnRecord { + TurnRecord { + v: 1, + source: SourceKind::ClaudeCode, + session_id: "sess-unpriced".into(), + session_path: None, + message_id: msg.into(), + turn_index: idx, + ts: ts.into(), + model: model.into(), + project: None, + project_key: None, + usage: Usage { + input: 100 + idx, + output: 50, + reasoning: 0, + cache_read: 0, + cache_create_5m: 0, + cache_create_1h: 0, + }, + tool_calls: vec![], + files_touched: None, + subagent: None, + stop_reason: None, + activity: None, + retries: None, + has_edits: None, + fidelity: None, + } + }; + + handle + .raw_mut() + .append_turns(&[ + make_turn( + 0, + "m-known", + "claude-sonnet-4-6", + "2026-05-01T00:00:00.000Z", + ), + make_turn( + 1, + "m-unknown-1", + "made-up-model-xyz", + "2026-05-01T00:01:00.000Z", + ), + make_turn( + 2, + "m-unknown-2", + "made-up-model-xyz", + "2026-05-01T00:02:00.000Z", + ), + ]) + .expect("append turns"); + + let report = handle + .summary_report(SummaryReportOptions::default()) + .expect("summary report"); + let SummaryReport::Grouped(grouped) = report else { + panic!("expected grouped report"); + }; + + assert_eq!(grouped.turn_count, 3); + assert_eq!( + grouped.unpriced_turns, 2, + "two turns used the unknown model" + ); + assert_eq!( + grouped.unpriced_models, + vec!["made-up-model-xyz"], + "unknown model listed exactly once" + ); + // The priced turn's cost must be non-zero; total must equal the priced portion. + assert!( + grouped.total_cost.total > 0.0, + "priced turn must contribute positive cost" + ); + } + /// Acceptance test for issue #437: the legacy `LedgerHandle::summary` /// surface (the slim one) also exposes the new counts. Verifies a turn /// without a stop_reason field round-trips to `None`/`none` rather diff --git a/memory/workspace/.relay/state.json b/memory/workspace/.relay/state.json index fdd7a063..7777cceb 100644 --- a/memory/workspace/.relay/state.json +++ b/memory/workspace/.relay/state.json @@ -1 +1 @@ -{"workspaceId":"rw_7ccfea89","remoteRoot":"/memory/workspace","localRoot":"/home/daytona/workspace/memory/workspace","mode":"poll","syncMode":"mirror","intervalMs":5000,"lastReconcileAt":"2026-06-10T16:21:00.371199852Z","lastSuccessfulReconcileAt":"2026-06-10T16:21:00.371199852Z","staleAfter":"2026-06-10T16:21:10.371199852Z","status":"ready","states":{"stale":false,"offline":false,"hasConflicts":false,"hasPendingWriteback":false},"pendingWriteback":0,"pendingConflicts":0,"deniedPaths":0,"counters":{"snapshotDeleteBlocked":157},"circuit":{"open":false,"openedAt":"0001-01-01T00:00:00Z","windowMs":60000,"cooldownMs":30000,"threshold":5,"nextRetry":"0001-01-01T00:00:00Z"},"outbox":{"pending":0,"needsAttention":0,"failed":0,"acked":0}} \ No newline at end of file +{"workspaceId":"rw_7ccfea89","remoteRoot":"/memory/workspace","localRoot":"/home/daytona/workspace/memory/workspace","mode":"poll","syncMode":"mirror","intervalMs":5000,"lastReconcileAt":"2026-06-10T16:21:00.371199852Z","lastSuccessfulReconcileAt":"2026-06-10T16:21:00.371199852Z","staleAfter":"2026-06-10T16:21:10.371199852Z","status":"ready","states":{"stale":false,"offline":false,"hasConflicts":false,"hasPendingWriteback":false},"pendingWriteback":0,"pendingConflicts":0,"deniedPaths":0,"counters":{"snapshotDeleteBlocked":157},"circuit":{"open":false,"openedAt":"0001-01-01T00:00:00Z","windowMs":60000,"cooldownMs":30000,"threshold":5,"nextRetry":"0001-01-01T00:00:00Z"},"outbox":{"pending":0,"needsAttention":0,"failed":0,"acked":0}} diff --git a/packages/sdk-node/src/index.d.ts b/packages/sdk-node/src/index.d.ts index 1d3558da..77881e00 100644 --- a/packages/sdk-node/src/index.d.ts +++ b/packages/sdk-node/src/index.d.ts @@ -114,6 +114,10 @@ export declare function summary(opts?: SummaryOptions): Promise<{ estimatedTokensSaved: number | bigint; }>; }; + /** Number of turns whose model had no pricing entry; their cost is reported as $0. */ + unpricedTurns?: number; + /** Distinct model names (first-seen order) that had no pricing entry. */ + unpricedModels?: string[]; }> export interface SessionCostOptions {