From 4323528166cef42ec6de6784a79192bddec9f840 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 15 Oct 2025 21:02:48 +0800 Subject: [PATCH 1/5] feat: forest-tool state-compute --- src/tool/main.rs | 1 + src/tool/subcommands/api_cmd.rs | 2 +- .../subcommands/api_cmd/api_compare_tests.rs | 2 +- .../api_cmd/generate_test_snapshot.rs | 4 +- src/tool/subcommands/mod.rs | 3 + src/tool/subcommands/state_compute_cmd.rs | 70 +++++++++++++++++++ 6 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 src/tool/subcommands/state_compute_cmd.rs diff --git a/src/tool/main.rs b/src/tool/main.rs index 0a1d76235d34..ec01a4301a93 100644 --- a/src/tool/main.rs +++ b/src/tool/main.rs @@ -36,6 +36,7 @@ where Subcommand::Api(cmd) => cmd.run().await, Subcommand::Net(cmd) => cmd.run().await, Subcommand::Shed(cmd) => cmd.run(client).await, + Subcommand::StateCompute(cmd) => cmd.run().await, Subcommand::Completion(cmd) => cmd.run(&mut std::io::stdout()), } }) diff --git a/src/tool/subcommands/api_cmd.rs b/src/tool/subcommands/api_cmd.rs index 09331bc44980..1a604f909518 100644 --- a/src/tool/subcommands/api_cmd.rs +++ b/src/tool/subcommands/api_cmd.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0, MIT mod api_compare_tests; -mod generate_test_snapshot; +pub(super) mod generate_test_snapshot; mod report; mod state_decode_params_tests; mod stateful_tests; diff --git a/src/tool/subcommands/api_cmd/api_compare_tests.rs b/src/tool/subcommands/api_cmd/api_compare_tests.rs index f280812e3736..96144554c724 100644 --- a/src/tool/subcommands/api_cmd/api_compare_tests.rs +++ b/src/tool/subcommands/api_cmd/api_compare_tests.rs @@ -143,7 +143,7 @@ impl TestSummary { /// Data about a failed test. Used for debugging. #[derive(Debug, Clone, Serialize, Deserialize)] -pub(super) struct TestDump { +pub struct TestDump { pub request: rpc::Request, pub forest_response: Result, pub lotus_response: Result, diff --git a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs index 995351672027..7bb761cbdfe9 100644 --- a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs @@ -70,9 +70,7 @@ pub async fn run_test_with_dump( Ok(()) } -pub(super) fn load_db( - db_root: &Path, -) -> anyhow::Result>>> { +pub fn load_db(db_root: &Path) -> anyhow::Result>>> { let db_writer = open_db(db_root.into(), &Default::default())?; let db = ManyCar::new(db_writer); let forest_car_db_dir = db_root.join(CAR_DB_DIR_NAME); diff --git a/src/tool/subcommands/mod.rs b/src/tool/subcommands/mod.rs index ece1af08367a..c84f7f8f6b93 100644 --- a/src/tool/subcommands/mod.rs +++ b/src/tool/subcommands/mod.rs @@ -12,6 +12,7 @@ mod index_cmd; mod net_cmd; mod shed_cmd; mod snapshot_cmd; +mod state_compute_cmd; mod state_migration_cmd; use crate::cli_shared::cli::*; @@ -81,5 +82,7 @@ pub enum Subcommand { #[command(subcommand)] Shed(shed_cmd::ShedCommands), + StateCompute(state_compute_cmd::StateComputeCommand), + Completion(CompletionCommand), } diff --git a/src/tool/subcommands/state_compute_cmd.rs b/src/tool/subcommands/state_compute_cmd.rs new file mode 100644 index 000000000000..568502d781b6 --- /dev/null +++ b/src/tool/subcommands/state_compute_cmd.rs @@ -0,0 +1,70 @@ +// Copyright 2019-2025 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::{ + cli_shared::{chain_path, read_config}, + daemon::db_util::load_all_forest_cars, + db::{ + CAR_DB_DIR_NAME, + car::ManyCar, + db_engine::{db_root, open_db}, + }, + networks::NetworkChain, + shim::clock::ChainEpoch, + tool::subcommands::api_cmd::generate_test_snapshot::ReadOpsTrackingStore, +}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +/// Compute state tree for an epoch +#[derive(Debug, clap::Args)] +pub struct StateComputeCommand { + /// Which epoch to compute the state transition for + #[arg(long, required = true)] + epoch: ChainEpoch, + /// Filecoin network chain + #[arg(long, required = true)] + chain: NetworkChain, + /// Optional path to the database folder or a snapshot `CAR` file + #[arg(long)] + db: Option, + /// Optional path to the database snapshot `CAR` file to write to for reproducing the computation + #[arg(long)] + export_db: Option, +} + +impl StateComputeCommand { + pub async fn run(self) -> anyhow::Result<()> { + let Self { + epoch, + chain, + db, + export_db, + } = self; + let db_root = if let Some(db) = db { + db + } else { + let (_, config) = read_config(None, Some(chain.clone()))?; + db_root(&chain_path(&config))? + }; + let (db, _tmp_dir) = if db_root.is_file() { + let temp_parity_db_root = tempfile::tempdir()?; + let db_writer = open_db(temp_parity_db_root.path().to_owned(), &Default::default())?; + let db = ManyCar::new(db_writer); + let forest_car_db_dir = db_root.join(CAR_DB_DIR_NAME); + load_all_forest_cars(&db, &forest_car_db_dir)?; + ( + Arc::new(ReadOpsTrackingStore::new(db)), + Some(temp_parity_db_root), + ) + } else { + ( + super::api_cmd::generate_test_snapshot::load_db(&db_root)?, + None, + ) + }; + Ok(()) + } +} From 365cdef535c002550894ccd1ddaf9559f882b228 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 17 Oct 2025 19:27:42 +0800 Subject: [PATCH 2/5] replay-compute --- docs/docs/users/reference/cli.sh | 7 +- src/db/car/any.rs | 9 +- src/tool/main.rs | 2 +- .../api_cmd/generate_test_snapshot.rs | 2 +- src/tool/subcommands/mod.rs | 3 +- src/tool/subcommands/state_compute_cmd.rs | 158 ++++++++++++++---- 6 files changed, 146 insertions(+), 35 deletions(-) diff --git a/docs/docs/users/reference/cli.sh b/docs/docs/users/reference/cli.sh index 1cc235b056b1..1c78c395224b 100755 --- a/docs/docs/users/reference/cli.sh +++ b/docs/docs/users/reference/cli.sh @@ -94,16 +94,21 @@ generate_markdown_section "forest-cli" "f3 ready" generate_markdown_section "forest-tool" "" generate_markdown_section "forest-tool" "backup" -generate_markdown_section "forest-tool" "completion" generate_markdown_section "forest-tool" "backup create" generate_markdown_section "forest-tool" "backup restore" +generate_markdown_section "forest-tool" "completion" + generate_markdown_section "forest-tool" "benchmark" generate_markdown_section "forest-tool" "benchmark car-streaming" generate_markdown_section "forest-tool" "benchmark graph-traversal" generate_markdown_section "forest-tool" "benchmark forest-encoding" generate_markdown_section "forest-tool" "benchmark export" +generate_markdown_section "forest-tool" "state" +generate_markdown_section "forest-tool" "state compute" +generate_markdown_section "forest-tool" "state replay-compute" + generate_markdown_section "forest-tool" "state-migration" generate_markdown_section "forest-tool" "state-migration actor-bundle" diff --git a/src/db/car/any.rs b/src/db/car/any.rs index f11bf40e8c68..1932eb19404e 100644 --- a/src/db/car/any.rs +++ b/src/db/car/any.rs @@ -18,7 +18,7 @@ use itertools::Either; use positioned_io::ReadAt; use std::borrow::Cow; use std::io::{Error, ErrorKind, Read, Result}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; pub enum AnyCar { @@ -138,6 +138,13 @@ impl TryFrom<&Path> for AnyCar { } } +impl TryFrom<&PathBuf> for AnyCar { + type Error = std::io::Error; + fn try_from(path: &PathBuf) -> std::io::Result { + Self::try_from(path.as_path()) + } +} + impl Blockstore for AnyCar where ReaderT: ReadAt, diff --git a/src/tool/main.rs b/src/tool/main.rs index ec01a4301a93..672cc084b527 100644 --- a/src/tool/main.rs +++ b/src/tool/main.rs @@ -36,7 +36,7 @@ where Subcommand::Api(cmd) => cmd.run().await, Subcommand::Net(cmd) => cmd.run().await, Subcommand::Shed(cmd) => cmd.run(client).await, - Subcommand::StateCompute(cmd) => cmd.run().await, + Subcommand::State(cmd) => cmd.run().await, Subcommand::Completion(cmd) => cmd.run(&mut std::io::stdout()), } }) diff --git a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs index 7bb761cbdfe9..f644de54c59c 100644 --- a/src/tool/subcommands/api_cmd/generate_test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/generate_test_snapshot.rs @@ -121,7 +121,7 @@ async fn ctx( db.clone(), db, chain_config, - genesis_header.clone(), + genesis_header, ) .unwrap(), ); diff --git a/src/tool/subcommands/mod.rs b/src/tool/subcommands/mod.rs index c84f7f8f6b93..7405d722103b 100644 --- a/src/tool/subcommands/mod.rs +++ b/src/tool/subcommands/mod.rs @@ -82,7 +82,8 @@ pub enum Subcommand { #[command(subcommand)] Shed(shed_cmd::ShedCommands), - StateCompute(state_compute_cmd::StateComputeCommand), + #[command(subcommand)] + State(state_compute_cmd::StateCommand), Completion(CompletionCommand), } diff --git a/src/tool/subcommands/state_compute_cmd.rs b/src/tool/subcommands/state_compute_cmd.rs index 568502d781b6..8a6a993fc046 100644 --- a/src/tool/subcommands/state_compute_cmd.rs +++ b/src/tool/subcommands/state_compute_cmd.rs @@ -2,69 +2,167 @@ // SPDX-License-Identifier: Apache-2.0, MIT use crate::{ + chain::{ChainStore, index::ResolveNullTipset}, cli_shared::{chain_path, read_config}, - daemon::db_util::load_all_forest_cars, db::{ - CAR_DB_DIR_NAME, - car::ManyCar, - db_engine::{db_root, open_db}, + MemoryDB, SettingsStoreExt, + car::{AnyCar, ManyCar}, + db_engine::db_root, }, - networks::NetworkChain, + genesis::read_genesis_header, + interpreter::VMTrace, + networks::{ChainConfig, NetworkChain}, shim::clock::ChainEpoch, - tool::subcommands::api_cmd::generate_test_snapshot::ReadOpsTrackingStore, -}; -use std::{ - path::{Path, PathBuf}, - sync::Arc, + state_manager::{StateManager, StateOutput}, }; +use std::{num::NonZeroUsize, path::PathBuf, sync::Arc, time::Instant}; + +/// Interact with Filecoin chain state +#[derive(Debug, clap::Subcommand)] +pub enum StateCommand { + Compute(ComputeCommand), + ReplayCompute(ReplayComputeCommand), +} + +impl StateCommand { + pub async fn run(self) -> anyhow::Result<()> { + match self { + Self::Compute(cmd) => cmd.run().await, + Self::ReplayCompute(cmd) => cmd.run().await, + } + } +} /// Compute state tree for an epoch #[derive(Debug, clap::Args)] -pub struct StateComputeCommand { +pub struct ComputeCommand { /// Which epoch to compute the state transition for #[arg(long, required = true)] epoch: ChainEpoch, /// Filecoin network chain #[arg(long, required = true)] chain: NetworkChain, - /// Optional path to the database folder or a snapshot `CAR` file + /// Optional path to the database folder #[arg(long)] db: Option, /// Optional path to the database snapshot `CAR` file to write to for reproducing the computation #[arg(long)] - export_db: Option, + export_db_to: Option, } -impl StateComputeCommand { +impl ComputeCommand { pub async fn run(self) -> anyhow::Result<()> { let Self { epoch, chain, db, - export_db, + export_db_to, } = self; - let db_root = if let Some(db) = db { + let db_root_path = if let Some(db) = db { db } else { let (_, config) = read_config(None, Some(chain.clone()))?; db_root(&chain_path(&config))? }; - let (db, _tmp_dir) = if db_root.is_file() { - let temp_parity_db_root = tempfile::tempdir()?; - let db_writer = open_db(temp_parity_db_root.path().to_owned(), &Default::default())?; - let db = ManyCar::new(db_writer); - let forest_car_db_dir = db_root.join(CAR_DB_DIR_NAME); - load_all_forest_cars(&db, &forest_car_db_dir)?; - ( - Arc::new(ReadOpsTrackingStore::new(db)), - Some(temp_parity_db_root), + let db = super::api_cmd::generate_test_snapshot::load_db(&db_root_path)?; + let chain_config = Arc::new(ChainConfig::from_chain(&chain)); + let genesis_header = + read_genesis_header(None, chain_config.genesis_bytes(&db).await?.as_deref(), &db) + .await?; + let chain_store = Arc::new( + ChainStore::new( + db.clone(), + db.clone(), + db.clone(), + db.clone(), + chain_config, + genesis_header, ) - } else { - ( - super::api_cmd::generate_test_snapshot::load_db(&db_root)?, - None, + .unwrap(), + ); + let ts = chain_store.chain_index().tipset_by_height( + epoch, + chain_store.heaviest_tipset(), + ResolveNullTipset::TakeOlder, + )?; + let epoch = ts.epoch(); + SettingsStoreExt::write_obj(&db.tracker, crate::db::setting_keys::HEAD_KEY, ts.key())?; + let state_manager = Arc::new(StateManager::new(chain_store.clone()).unwrap()); + + let StateOutput { + state_root, + receipt_root, + .. + } = state_manager + .compute_tipset_state(ts, crate::state_manager::NO_CALLBACK, VMTrace::NotTraced) + .await?; + let mut db_snapshot = vec![]; + db.export_forest_car(&mut db_snapshot).await?; + println!( + "epoch: {epoch}, state_root: {state_root}, receipt_root: {receipt_root}, db_snapshot_size: {}", + human_bytes::human_bytes(db_snapshot.len() as f64) + ); + if let Some(export_db_to) = export_db_to { + std::fs::write(export_db_to, db_snapshot)?; + } + Ok(()) + } +} + +/// Replay state computation with a db snapshot +#[derive(Debug, clap::Args)] +pub struct ReplayComputeCommand { + /// Path to the database snapshot `CAR` file generated by `forest-tool state compute` + snapshot: PathBuf, + /// Filecoin network chain + #[arg(long, required = true)] + chain: NetworkChain, + /// Number of times to repeat the state computation + #[arg(short, long, default_value_t = NonZeroUsize::new(1).unwrap())] + n: NonZeroUsize, +} + +impl ReplayComputeCommand { + pub async fn run(self) -> anyhow::Result<()> { + let Self { snapshot, chain, n } = self; + let snap_car = AnyCar::try_from(&snapshot)?; + let ts = Arc::new(snap_car.heaviest_tipset()?); + let epoch = ts.epoch(); + let db = Arc::new(ManyCar::new(MemoryDB::default()).with_read_only(snap_car)?); + let chain_config = Arc::new(ChainConfig::from_chain(&chain)); + let genesis_header = + read_genesis_header(None, chain_config.genesis_bytes(&db).await?.as_deref(), &db) + .await?; + let chain_store = Arc::new( + ChainStore::new( + db.clone(), + db.clone(), + db.clone(), + db.clone(), + chain_config, + genesis_header, ) - }; + .unwrap(), + ); + let state_manager = Arc::new(StateManager::new(chain_store.clone()).unwrap()); + for _ in 0..n.get() { + let start = Instant::now(); + let StateOutput { + state_root, + receipt_root, + .. + } = state_manager + .compute_tipset_state( + ts.clone(), + crate::state_manager::NO_CALLBACK, + VMTrace::NotTraced, + ) + .await?; + println!( + "epoch: {epoch}, state_root: {state_root}, receipt_root: {receipt_root}, took {}.", + humantime::format_duration(start.elapsed()) + ); + } Ok(()) } } From e207be1320e3743b813a1ad3afc1142c01bc5de9 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 17 Oct 2025 20:25:19 +0800 Subject: [PATCH 3/5] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56bd8621a5c1..f458eb468009 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,10 @@ - [#6166](https://github.com/ChainSafe/forest/pull/6166) Gate `JWT` expiration validation behind environment variable `FOREST_JWT_DISABLE_EXP_VALIDATION`. +- [#6167](https://github.com/ChainSafe/forest/pull/6167) Added `forest-tool state compute` subcommand to generate database snapshot for tipset validation. + +- [#6167](https://github.com/ChainSafe/forest/pull/6167) Added `forest-tool state replay-compute` subcommand to replay tipset validation with a minimal database snapshot. + ### Changed - [#6145](https://github.com/ChainSafe/forest/pull/6145) Updated `forest-cli snapshot export` to use v2 format by default. From 78dff806bf110c2db36e6f1b1c16e5f89b203e29 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 24 Oct 2025 17:55:27 +0800 Subject: [PATCH 4/5] Update src/tool/subcommands/state_compute_cmd.rs Co-authored-by: Hubert --- src/tool/subcommands/state_compute_cmd.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tool/subcommands/state_compute_cmd.rs b/src/tool/subcommands/state_compute_cmd.rs index 8a6a993fc046..9f7a709615ee 100644 --- a/src/tool/subcommands/state_compute_cmd.rs +++ b/src/tool/subcommands/state_compute_cmd.rs @@ -110,6 +110,7 @@ impl ComputeCommand { } /// Replay state computation with a db snapshot +/// To be used in conjunction with `forest-tool state compute`. #[derive(Debug, clap::Args)] pub struct ReplayComputeCommand { /// Path to the database snapshot `CAR` file generated by `forest-tool state compute` From d0c08c67ab9f1345bac4e7bcb1088b60622debef Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 24 Oct 2025 18:02:56 +0800 Subject: [PATCH 5/5] resolve comments --- src/tool/subcommands/state_compute_cmd.rs | 42 ++++++++++------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/src/tool/subcommands/state_compute_cmd.rs b/src/tool/subcommands/state_compute_cmd.rs index 9f7a709615ee..c3684ec9e470 100644 --- a/src/tool/subcommands/state_compute_cmd.rs +++ b/src/tool/subcommands/state_compute_cmd.rs @@ -69,17 +69,14 @@ impl ComputeCommand { let genesis_header = read_genesis_header(None, chain_config.genesis_bytes(&db).await?.as_deref(), &db) .await?; - let chain_store = Arc::new( - ChainStore::new( - db.clone(), - db.clone(), - db.clone(), - db.clone(), - chain_config, - genesis_header, - ) - .unwrap(), - ); + let chain_store = Arc::new(ChainStore::new( + db.clone(), + db.clone(), + db.clone(), + db.clone(), + chain_config, + genesis_header, + )?); let ts = chain_store.chain_index().tipset_by_height( epoch, chain_store.heaviest_tipset(), @@ -87,7 +84,7 @@ impl ComputeCommand { )?; let epoch = ts.epoch(); SettingsStoreExt::write_obj(&db.tracker, crate::db::setting_keys::HEAD_KEY, ts.key())?; - let state_manager = Arc::new(StateManager::new(chain_store.clone()).unwrap()); + let state_manager = Arc::new(StateManager::new(chain_store.clone())?); let StateOutput { state_root, @@ -134,18 +131,15 @@ impl ReplayComputeCommand { let genesis_header = read_genesis_header(None, chain_config.genesis_bytes(&db).await?.as_deref(), &db) .await?; - let chain_store = Arc::new( - ChainStore::new( - db.clone(), - db.clone(), - db.clone(), - db.clone(), - chain_config, - genesis_header, - ) - .unwrap(), - ); - let state_manager = Arc::new(StateManager::new(chain_store.clone()).unwrap()); + let chain_store = Arc::new(ChainStore::new( + db.clone(), + db.clone(), + db.clone(), + db.clone(), + chain_config, + genesis_header, + )?); + let state_manager = Arc::new(StateManager::new(chain_store.clone())?); for _ in 0..n.get() { let start = Instant::now(); let StateOutput {