Skip to content

relayburn-sdk: bootstrap burn.sqlite from ledger.jsonl on Ledger::open (α-followup) #357

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
willwashburn merged 1 commit into from
May 7, 2026
Merged

relayburn-sdk: bootstrap burn.sqlite from ledger.jsonl on Ledger::open (α-followup) #357

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions crates/relayburn-cli/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,8 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync"] }
# capture and the Rust diff runner. `serde` / `serde_json` are workspace deps
# already pulled in via `[dependencies]`, so this entry is documentation only.
#
# The diff runner also bootstraps the SQLite fixture from the committed
# `ledger.jsonl` (the in-tree fixture is JSONL-only because the SQLite
# binaries are gitignored). It does so by parsing the JSONL with `serde_json`
# and replaying the records via `relayburn_sdk::RawLedger::append_*` methods,
# which is reachable through the `relayburn-sdk` dep already declared above.
# The in-tree fixture is JSONL-only (the SQLite binaries are gitignored); the
# diff runner just deletes any prior sqlite before invoking the binary, and
# the SDK's `Ledger::open` rebuilds `burn.sqlite` from `ledger.jsonl`
# automatically (see `relayburn_sdk::ledger::bootstrap`). No JSONL parsing in
# the test helper itself.
167 changes: 22 additions & 145 deletions crates/relayburn-cli/tests/golden.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,38 +36,28 @@ use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;

use relayburn_sdk::{
CompactionEvent, RawLedger, SessionRelationshipRecord, Stamp, ToolResultEventRecord,
TurnRecord, UserTurnRecord,
};
use serde::Deserialize;

/// Bootstrap the SQLite fixture from the committed `ledger.jsonl`.
/// Wipe any prior `burn.sqlite` / `content.sqlite` so the next
/// `Ledger::open` deterministically rebuilds from `ledger.jsonl`.
///
/// The CLI-golden fixture's source of truth is `ledger.jsonl` (the SQLite
/// counterparts are gitignored because they're rematerialized on demand;
/// see `tests/fixtures/cli-golden/ledger/.gitignore`). The TS CLI reads
/// JSONL natively via its `file` storage adapter; the Rust SDK is sqlite-
/// only, so we replay the JSONL into `burn.sqlite` here before invoking
/// the binary.
/// The CLI-golden fixture's source of truth is `ledger.jsonl` (the
/// SQLite counterparts are gitignored because they're rematerialized
/// on demand; see `tests/fixtures/cli-golden/ledger/.gitignore`). The
/// TS CLI reads JSONL natively via its `file` storage adapter; the Rust
/// SDK is sqlite-only and bootstraps `burn.sqlite` from the JSONL on
/// open (see `relayburn_sdk::ledger::bootstrap`).
///
/// Idempotent: if `burn.sqlite` already has a non-empty `turns` table we
/// skip the rebuild. Local devs running the diff runner repeatedly thus
/// only pay the JSONL-replay cost once.
fn bootstrap_sqlite_from_jsonl(ledger_home: &Path) -> std::io::Result<()> {
let jsonl_path = ledger_home.join("ledger.jsonl");
if !jsonl_path.is_file() {
// No JSONL source: assume the fixture was built another way and
// bail (the binary will surface the resulting empty-ledger
// diff loud and clear when it runs).
/// We could rely on the SDK's mtime check to do this for free, but a
/// stale sqlite from a prior run with a *newer* mtime than the JSONL
/// would otherwise mask snapshot drift. Wiping forces a fresh replay
/// every test run.
fn reset_sqlite_for_fresh_bootstrap(ledger_home: &Path) -> std::io::Result<()> {
if !ledger_home.join("ledger.jsonl").is_file() {
// No JSONL source — leave whatever sqlite is here alone and
// let the binary surface any resulting empty-ledger diff.
return Ok(());
}
let burn_path = ledger_home.join("burn.sqlite");
let content_path = ledger_home.join("content.sqlite");

// Wipe any prior bootstrap. We don't try to do incremental upserts
// here — the JSONL is canonical and small; rewriting from scratch
// keeps the bootstrap deterministic across runs.
for name in [
"burn.sqlite",
"burn.sqlite-shm",
Expand All @@ -76,125 +66,11 @@ fn bootstrap_sqlite_from_jsonl(ledger_home: &Path) -> std::io::Result<()> {
"content.sqlite-shm",
"content.sqlite-wal",
] {
let p = ledger_home.join(name);
let _ = fs::remove_file(p);
}

let mut ledger =
RawLedger::open(&burn_path, &content_path).expect("open fixture ledger for bootstrap");

let raw = fs::read_to_string(&jsonl_path)?;
let mut turns: Vec<TurnRecord> = Vec::new();
let mut user_turns: Vec<UserTurnRecord> = Vec::new();
let mut tool_results: Vec<ToolResultEventRecord> = Vec::new();
let mut relationships: Vec<SessionRelationshipRecord> = Vec::new();
let mut compactions: Vec<CompactionEvent> = Vec::new();
let mut stamps: Vec<Stamp> = Vec::new();

for (line_no, line) in raw.lines().enumerate() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let envelope: serde_json::Value = serde_json::from_str(trimmed).unwrap_or_else(|err| {
panic!(
"[golden bootstrap] line {} of ledger.jsonl is not valid JSON: {err}",
line_no + 1
)
});
let kind = envelope.get("kind").and_then(|v| v.as_str()).unwrap_or("");
let mut record = envelope
.get("record")
.cloned()
.unwrap_or(serde_json::Value::Null);
match kind {
"turn" => turns.push(serde_json::from_value(record).expect("turn record")),
"user_turn" => {
user_turns.push(serde_json::from_value(record).expect("user_turn record"))
}
"tool_result_event" => {
normalize_tool_result_event(&mut record);
tool_results.push(serde_json::from_value(record).expect("tool_result_event record"))
}
"relationship" => {
relationships.push(serde_json::from_value(record).expect("relationship record"))
}
"compaction" => {
compactions.push(serde_json::from_value(record).expect("compaction record"))
}
"stamp" => stamps.push(stamp_from_envelope(&envelope)),
_ => {
// Unknown kinds (`text`, `tool_result`, etc. emitted by older
// ledger writers) are noise here — they belong to the content
// sidecar lifecycle, not the events DB.
}
}
}

ledger.append_turns(&turns).expect("append turns");
ledger
.append_user_turns(&user_turns)
.expect("append user_turns");
ledger
.append_tool_result_events(&tool_results)
.expect("append tool_result_events");
ledger
.append_relationships(&relationships)
.expect("append relationships");
ledger
.append_compactions(&compactions)
.expect("append compactions");
for s in &stamps {
ledger.append_stamp(s).expect("append stamp");
let _ = fs::remove_file(ledger_home.join(name));
}
Ok(())
}

/// The hand-built fixture writes `eventSource: "transcript"` for Claude
/// `tool_result` events; the canonical schema dropped that variant in
/// favor of the more specific `"tool_result"` value. The TS reader is
/// lenient and stores the JSON verbatim; the Rust SDK is strict. Normalize
/// here so the fixture replays cleanly without retroactively rewriting
/// the JSONL on disk (which would drift the snapshot capture corpus). The
/// substitution also fills in `eventIndex` if the fixture omits it
/// (required by the SDK schema; the TS reader defaults missing values to
/// `0` via `??`).
fn normalize_tool_result_event(record: &mut serde_json::Value) {
let Some(obj) = record.as_object_mut() else {
return;
};
if let Some(src) = obj.get_mut("eventSource") {
if src.as_str() == Some("transcript") {
*src = serde_json::Value::String("tool_result".to_string());
}
}
obj.entry("eventIndex").or_insert(serde_json::json!(0));
}

fn stamp_from_envelope(envelope: &serde_json::Value) -> Stamp {
Stamp {
ts: envelope
.get("ts")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
selector: serde_json::from_value(
envelope
.get("selector")
.cloned()
.unwrap_or(serde_json::Value::Null),
)
.unwrap_or_default(),
enrichment: serde_json::from_value(
envelope
.get("enrichment")
.cloned()
.unwrap_or(serde_json::Value::Null),
)
.unwrap_or_default(),
}
}

#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct Invocation {
Expand Down Expand Up @@ -256,10 +132,11 @@ fn golden_diff_against_ts_cli_snapshots() {
let ledger_home = fixture_dir.join("ledger");
let project_dir = fixture_dir.join("project");

// Bootstrap the SQLite fixture from `ledger.jsonl`. The Rust SDK only
// reads from sqlite; the in-tree fixture is JSONL-only because the
// sqlite binaries are gitignored. Replays once per test run.
bootstrap_sqlite_from_jsonl(&ledger_home).expect("bootstrap sqlite from JSONL");
// The in-tree fixture is JSONL-only (the sqlite binaries are
// gitignored). Wipe any prior sqlite so the SDK's bootstrap-on-open
// (see `relayburn_sdk::ledger::bootstrap`) deterministically replays
// the JSONL on the binary's first `Ledger::open`.
reset_sqlite_for_fresh_bootstrap(&ledger_home).expect("reset sqlite for fresh bootstrap");

// Sealed HOME so the Rust binary's eventual ingest sweep doesn't
// discover the developer's real session stores.
Expand Down
1 change: 1 addition & 0 deletions crates/relayburn-sdk/src/ledger.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
// agent absorbing more verbs will need them.
#![allow(dead_code, unused_imports)]

mod bootstrap;
mod config;
mod content;
mod db;
Expand Down
Loading
Loading