Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ Cross-package release notes for relayburn. Package changelogs contain package-le

## [Unreleased]

- `burn summary` now reports turns whose model has no pricing entry (`unpricedTurns`/`unpricedModels` in JSON output, warning footer in human output) instead of silently counting them at $0.

## [3.2.1] - 2026-06-09

### Added
Expand Down
15 changes: 15 additions & 0 deletions crates/relayburn-cli/src/commands/summary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1014,6 +1014,17 @@ fn emit_human(report: &SummaryGroupedReport, ingest_report: &relayburn_sdk::Inge
let out = lines.join("\n");
// TS uses `process.stdout.write(lines.join('\n'))` — no trailing newline.
print!("{}", out);

if report.unpriced_turns > 0 {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Include unpriced fields in CLI JSON output

When burn summary --json is used with turns from an unknown model, this new warning path is skipped because emit_grouped returns through emit_json, and grouped_json_value manually builds the payload without inserting unpricedTurns or unpricedModels. That leaves JSON callers with the same silent $0 under-reporting this change is intended to surface; add the two fields to the grouped JSON payload when present.

Useful? React with 👍 / 👎.

let models = report.unpriced_models.join(", ");
eprintln!(
"warning: {} turn(s) had no pricing for model(s): {} — their cost is reported as $0.",
report.unpriced_turns, models,
);
eprintln!(
" Update the snapshot (pnpm run pricing:update) or add an override at $RELAYBURN_HOME/models.dev.json.",
);
}
}

fn render_quality(q: &QualityResult) -> String {
Expand Down Expand Up @@ -1271,6 +1282,8 @@ mod tests {
stop_reasons: relayburn_sdk::StopReasonCounts::default(),
subagents: SubagentCounts::default(),
quality: Some(QualityResult::default()),
unpriced_turns: 0,
unpriced_models: Vec::new(),
};

let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty());
Expand Down Expand Up @@ -1324,6 +1337,8 @@ mod tests {
stop_reasons: relayburn_sdk::StopReasonCounts::default(),
subagents: SubagentCounts::default(),
quality: None,
unpriced_turns: 0,
unpriced_models: Vec::new(),
};
let value = grouped_json_value(&report, &relayburn_sdk::IngestReport::empty());
assert!(
Expand Down
2 changes: 1 addition & 1 deletion crates/relayburn-sdk/src/analyze.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ pub use context_delta::{
deltas_for_session, ContextDelta, ContextDeltaOpts, InterveningStep, OwnerFilter, OwnerRail,
ReminderSource,
};
pub use cost::{cost_for_turn, cost_for_usage, sum_costs, CostBreakdown};
pub use cost::{cost_for_turn, cost_for_usage, sum_costs, tally_unpriced, CostBreakdown};
pub use fidelity::{
has_minimum_fidelity, summarize_fidelity, summarize_fidelity_from_iter, FidelitySummary,
};
Expand Down
67 changes: 67 additions & 0 deletions crates/relayburn-sdk/src/analyze/cost.rs
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,23 @@ fn strip_provider_prefix(model: &str) -> &str {
}
}

/// Count turns whose model has no pricing entry, and collect the distinct
/// model names, first-seen order. Used by summary surfaces to make pricing
/// gaps visible instead of silently folding them in at $0.
pub fn tally_unpriced(turns: &[TurnRecord], pricing: &PricingTable) -> (u64, Vec<String>) {
let mut count = 0u64;
let mut models: Vec<String> = Vec::new();
for t in turns {
if lookup_model_rate(&t.model, pricing).is_none() {
count += 1;
if !models.iter().any(|m| m == &t.model) {
models.push(t.model.clone());
}
}
}
(count, models)
}

pub fn sum_costs<I, B>(costs: I) -> CostBreakdown
where
I: IntoIterator<Item = B>,
Expand Down Expand Up @@ -585,6 +602,56 @@ mod tests {
assert_eq!(s.output, 1.5);
}

#[test]
fn tally_unpriced_returns_zero_when_all_models_priced() {
let p = load_builtin_pricing();
let turns = vec![
turn(
"claude-sonnet-4-6",
usage_with(100, 50, 0),
SourceKind::ClaudeCode,
),
turn(
"claude-opus-4-7",
usage_with(200, 100, 0),
SourceKind::ClaudeCode,
),
];
let (count, models) = tally_unpriced(&turns, &p);
assert_eq!(count, 0);
assert!(models.is_empty());
}

#[test]
fn tally_unpriced_counts_turns_and_deduplicates_models() {
let p = load_builtin_pricing();
// "made-up-model-xyz" is not in the pricing table; appears in two turns.
let turns = vec![
turn(
"made-up-model-xyz",
usage_with(100, 50, 0),
SourceKind::ClaudeCode,
),
turn(
"made-up-model-xyz",
usage_with(200, 100, 0),
SourceKind::ClaudeCode,
),
turn(
"claude-sonnet-4-6",
usage_with(300, 150, 0),
SourceKind::ClaudeCode,
),
];
let (count, models) = tally_unpriced(&turns, &p);
assert_eq!(count, 2, "two turns used the unknown model");
assert_eq!(
models,
vec!["made-up-model-xyz"],
"model listed exactly once"
);
}

#[test]
fn sum_costs_on_empty_input_returns_zero_aggregate() {
let empty: Vec<CostBreakdown> = vec![];
Expand Down
185 changes: 175 additions & 10 deletions crates/relayburn-sdk/src/query_verbs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,17 @@ use crate::analyze::{
load_claude_settings, load_overhead_file, load_pricing, project_claude_settings_path,
provider_for, render_unified_diff_for_recommendation, sort_findings, sum_costs,
summarize_fidelity, summarize_fidelity_from_iter, summarize_replacement_savings,
tool_call_pattern_to_finding, tool_output_bloat_to_finding, user_claude_settings_path,
AggregateByProviderOptions, AttributeOverheadInput, AttributionMethod, BashAggregation,
BashVerbAggregation, BuildSubagentTreeOptions, CompareOptions as AnalyzeCompareOptions,
CompareTable, ComputeQualityOptions, CostBreakdown, CoverageField, DetectPatternsOptions,
DetectToolCallPatternsOptions, DetectToolOutputBloatOptions, FidelitySummary, FieldCoverage,
FileAggregation, GhostSurfaceFindingOptions, HotspotsOptions as AnalyzeHotspotsOptions,
LoadedClaudeSettings, MarkdownSection, McpServerAggregation, OverheadFile, OverheadFileKind,
ParsedOverheadFile, PricingTable, ProviderAggregateRow, ProviderFilter, QualityResult,
ReplacementSavingsSummary, RowCoverage, SessionClaudeMdCost, SubagentAggregation,
SubagentTreeNode, SubagentTypeStats, ToolSavingsAggregate, UsageCostAggregateRow, WasteFinding,
tally_unpriced, tool_call_pattern_to_finding, tool_output_bloat_to_finding,
user_claude_settings_path, AggregateByProviderOptions, AttributeOverheadInput,
AttributionMethod, BashAggregation, BashVerbAggregation, BuildSubagentTreeOptions,
CompareOptions as AnalyzeCompareOptions, CompareTable, ComputeQualityOptions, CostBreakdown,
CoverageField, DetectPatternsOptions, DetectToolCallPatternsOptions,
DetectToolOutputBloatOptions, FidelitySummary, FieldCoverage, FileAggregation,
GhostSurfaceFindingOptions, HotspotsOptions as AnalyzeHotspotsOptions, LoadedClaudeSettings,
MarkdownSection, McpServerAggregation, OverheadFile, OverheadFileKind, ParsedOverheadFile,
PricingTable, ProviderAggregateRow, ProviderFilter, QualityResult, ReplacementSavingsSummary,
RowCoverage, SessionClaudeMdCost, SubagentAggregation, SubagentTreeNode, SubagentTypeStats,
ToolSavingsAggregate, UsageCostAggregateRow, WasteFinding,
};
use crate::ledger::{EnrichedTurn, Enrichment, Query};
use crate::reader::{
Expand Down Expand Up @@ -496,6 +497,14 @@ pub struct Summary {
/// Counts roll up the trailing `stop_reason` of every assistant turn
/// in the filtered slice. See #437.
pub stop_reasons: StopReasonCounts,
/// Count of turns whose model had no entry in the pricing snapshot.
/// Their cost is reported as $0. Zero when all models are priced.
#[serde(default)]
pub unpriced_turns: u64,
/// Distinct model names (first-seen order) that had no pricing entry.
/// Empty when all models are priced.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub unpriced_models: Vec<String>,
}

impl LedgerHandle {
Expand Down Expand Up @@ -602,6 +611,10 @@ fn compute_summary(turns: &[TurnRecord], pricing: &PricingTable) -> Summary {
None
};

// Use the same pricing table that was used for cost accumulation so the
// count precisely matches which turns contributed $0 to `total_cost`.
let (unpriced_turns, unpriced_models) = tally_unpriced(turns, pricing);

Summary {
total_tokens,
total_cost,
Expand All @@ -617,6 +630,8 @@ fn compute_summary(turns: &[TurnRecord], pricing: &PricingTable) -> Summary {
by_tag: None,
replacement_savings,
stop_reasons: StopReasonCounts::from_turns(turns),
unpriced_turns,
unpriced_models,
}
}

Expand Down Expand Up @@ -792,6 +807,14 @@ pub struct SummaryGroupedReport {
pub subagents: crate::reader::SubagentCounts,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub quality: Option<QualityResult>,
/// Count of turns whose model had no entry in the pricing snapshot.
/// Their cost is reported as $0. Zero when all models are priced.
#[serde(default)]
pub unpriced_turns: u64,
/// Distinct model names (first-seen order) that had no pricing entry.
/// Empty when all models are priced.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub unpriced_models: Vec<String>,
}

#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
Expand Down Expand Up @@ -982,6 +1005,7 @@ impl LedgerHandle {
// behavior.
let session_filter = summary_subagent_session_filter(&opts, &turns);
let subagents = compute_summary_subagent_counts(session_filter.as_ref());
let (unpriced_turns, unpriced_models) = tally_unpriced(&turns, &pricing);
Ok(SummaryReport::Grouped(SummaryGroupedReport {
group_by,
tag_key,
Expand All @@ -995,6 +1019,8 @@ impl LedgerHandle {
stop_reasons,
subagents,
quality,
unpriced_turns,
unpriced_models,
}))
}
SummaryReportMode::ByTool => {
Expand Down Expand Up @@ -5046,6 +5072,145 @@ mod tests {
assert!(!grouped.stop_reasons.is_empty());
}

/// `compute_summary` (the slim legacy verb) populates unpriced_turns
/// and unpriced_models when a turn's model is absent from the pricing table.
#[test]
fn compute_summary_tracks_unpriced_turns_and_models() {
let pricing = load_pricing(None);
let unknown_model = "made-up-model-xyz";
let priced_turn = TurnRecord {
v: 1,
source: SourceKind::ClaudeCode,
session_id: "s".into(),
session_path: None,
message_id: "m-priced".into(),
turn_index: 0,
ts: "2026-05-01T00:00:00.000Z".into(),
model: "claude-sonnet-4-6".into(),
project: None,
project_key: None,
usage: Usage {
input: 100,
output: 50,
reasoning: 0,
cache_read: 0,
cache_create_5m: 0,
cache_create_1h: 0,
},
tool_calls: vec![],
files_touched: None,
subagent: None,
stop_reason: None,
activity: None,
retries: None,
has_edits: None,
fidelity: None,
};
let unpriced_turn = TurnRecord {
message_id: "m-unpriced".into(),
turn_index: 1,
ts: "2026-05-01T00:01:00.000Z".into(),
model: unknown_model.into(),
..priced_turn.clone()
};
let turns = vec![priced_turn, unpriced_turn];
let s = compute_summary(&turns, &pricing);
assert_eq!(s.turn_count, 2);
assert_eq!(s.unpriced_turns, 1, "one turn uses an unknown model");
assert_eq!(
s.unpriced_models,
vec![unknown_model],
"unknown model listed exactly once"
);
}

/// `summary_report` (grouped mode) surfaces unpriced turn count and model
/// names when a turn's model is absent from the pricing snapshot.
#[test]
fn summary_report_grouped_tracks_unpriced_turns_and_models() {
let dir = tempfile::tempdir().unwrap();
let opts = LedgerOpenOptions::with_home(dir.path());
let mut handle = Ledger::open(opts).expect("open ledger");

let make_turn = |idx: u64, msg: &str, model: &str, ts: &str| -> TurnRecord {
TurnRecord {
v: 1,
source: SourceKind::ClaudeCode,
session_id: "sess-unpriced".into(),
session_path: None,
message_id: msg.into(),
turn_index: idx,
ts: ts.into(),
model: model.into(),
project: None,
project_key: None,
usage: Usage {
input: 100 + idx,
output: 50,
reasoning: 0,
cache_read: 0,
cache_create_5m: 0,
cache_create_1h: 0,
},
tool_calls: vec![],
files_touched: None,
subagent: None,
stop_reason: None,
activity: None,
retries: None,
has_edits: None,
fidelity: None,
}
};

handle
.raw_mut()
.append_turns(&[
make_turn(
0,
"m-known",
"claude-sonnet-4-6",
"2026-05-01T00:00:00.000Z",
),
make_turn(
1,
"m-unknown-1",
"made-up-model-xyz",
"2026-05-01T00:01:00.000Z",
),
make_turn(
2,
"m-unknown-2",
"made-up-model-xyz",
"2026-05-01T00:02:00.000Z",
),
])
.expect("append turns");

let report = handle
.summary_report(SummaryReportOptions::default())
.expect("summary report");
let SummaryReport::Grouped(grouped) = report else {
panic!("expected grouped report");
};

assert_eq!(grouped.turn_count, 3);
assert_eq!(
grouped.unpriced_turns, 2,
"two turns used the unknown model"
);
assert_eq!(
grouped.unpriced_models,
vec!["made-up-model-xyz"],
"unknown model listed exactly once"
);
// The priced turn's cost must be non-zero; total must equal the priced portion.
assert!(
grouped.total_cost.total > 0.0,
"priced turn must contribute positive cost"
);
}

/// Acceptance test for issue #437: the legacy `LedgerHandle::summary`
/// surface (the slim one) also exposes the new counts. Verifies a turn
/// without a stop_reason field round-trips to `None`/`none` rather
Expand Down
2 changes: 1 addition & 1 deletion memory/workspace/.relay/state.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"workspaceId":"rw_7ccfea89","remoteRoot":"/memory/workspace","localRoot":"/home/daytona/workspace/memory/workspace","mode":"poll","syncMode":"mirror","intervalMs":5000,"lastReconcileAt":"2026-06-10T16:21:00.371199852Z","lastSuccessfulReconcileAt":"2026-06-10T16:21:00.371199852Z","staleAfter":"2026-06-10T16:21:10.371199852Z","status":"ready","states":{"stale":false,"offline":false,"hasConflicts":false,"hasPendingWriteback":false},"pendingWriteback":0,"pendingConflicts":0,"deniedPaths":0,"counters":{"snapshotDeleteBlocked":157},"circuit":{"open":false,"openedAt":"0001-01-01T00:00:00Z","windowMs":60000,"cooldownMs":30000,"threshold":5,"nextRetry":"0001-01-01T00:00:00Z"},"outbox":{"pending":0,"needsAttention":0,"failed":0,"acked":0}}
{"workspaceId":"rw_7ccfea89","remoteRoot":"/memory/workspace","localRoot":"/home/daytona/workspace/memory/workspace","mode":"poll","syncMode":"mirror","intervalMs":5000,"lastReconcileAt":"2026-06-10T16:21:00.371199852Z","lastSuccessfulReconcileAt":"2026-06-10T16:21:00.371199852Z","staleAfter":"2026-06-10T16:21:10.371199852Z","status":"ready","states":{"stale":false,"offline":false,"hasConflicts":false,"hasPendingWriteback":false},"pendingWriteback":0,"pendingConflicts":0,"deniedPaths":0,"counters":{"snapshotDeleteBlocked":157},"circuit":{"open":false,"openedAt":"0001-01-01T00:00:00Z","windowMs":60000,"cooldownMs":30000,"threshold":5,"nextRetry":"0001-01-01T00:00:00Z"},"outbox":{"pending":0,"needsAttention":0,"failed":0,"acked":0}}
4 changes: 4 additions & 0 deletions packages/sdk-node/src/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,10 @@ export declare function summary(opts?: SummaryOptions): Promise<{
estimatedTokensSaved: number | bigint;
}>;
};
/** Number of turns whose model had no pricing entry; their cost is reported as $0. */
unpricedTurns?: number;
/** Distinct model names (first-seen order) that had no pricing entry. */
unpricedModels?: string[];
Comment on lines +118 to +120

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Propagate unpriced fields through the Node binding

For @relayburn/sdk callers with unknown-model turns, these declarations promise unpricedTurns/unpricedModels, but the napi Summary object in crates/relayburn-sdk-node/src/lib.rs still has no such fields and its From<sdk::Summary> conversion drops them. The runtime result from summary() therefore omits the new signal while TypeScript suggests it may exist; extend the binding struct and conversion as well.

Useful? React with 👍 / 👎.

}>

export interface SessionCostOptions {
Expand Down
Loading