From b741a6d2592f549eef3a9615668a9611d52b6a99 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 14 Jan 2026 19:19:24 +0800 Subject: [PATCH 1/5] fix(rpc): backfill tipset messages from p2p network in a few RPC methods --- src/blocks/tipset.rs | 25 ------------------------- src/chain_sync/mod.rs | 2 +- src/rpc/methods/chain.rs | 25 +++++++++++++++---------- 3 files changed, 16 insertions(+), 36 deletions(-) diff --git a/src/blocks/tipset.rs b/src/blocks/tipset.rs index 33fc9e462d18..e43202cdfe22 100644 --- a/src/blocks/tipset.rs +++ b/src/blocks/tipset.rs @@ -281,31 +281,6 @@ impl Tipset { Tipset::load(store, tsk)?.context("Required tipset missing from database") } - /// Constructs and returns a full tipset if messages from storage exists - pub fn fill_from_blockstore(&self, store: &impl Blockstore) -> Option { - // Find tipset messages. If any are missing, return `None`. - let blocks = self - .block_headers() - .iter() - .cloned() - .map(|header| { - let (bls_messages, secp_messages) = - crate::chain::store::block_messages(store, &header).ok()?; - Some(Block { - header, - bls_messages, - secp_messages, - }) - }) - .collect::>>()?; - - // the given tipset has already been verified, so this cannot fail - Some( - FullTipset::new(blocks) - .expect("block headers have already been verified so this check cannot fail"), - ) - } - /// Returns epoch of the tipset. pub fn epoch(&self) -> ChainEpoch { self.min_ticket_block().epoch diff --git a/src/chain_sync/mod.rs b/src/chain_sync/mod.rs index 790204c69b1e..998a5835c965 100644 --- a/src/chain_sync/mod.rs +++ b/src/chain_sync/mod.rs @@ -13,7 +13,7 @@ mod validation; pub use self::{ bad_block_cache::BadBlockCache, - chain_follower::{ChainFollower, load_full_tipset}, + chain_follower::{ChainFollower, get_full_tipset, load_full_tipset}, chain_muxer::SyncConfig, consensus::collect_errs, sync_status::{ForkSyncInfo, ForkSyncStage, NodeSyncStatus, SyncStatus, SyncStatusReport}, diff --git a/src/rpc/methods/chain.rs b/src/rpc/methods/chain.rs index 02b1e9fbffe5..4d98af65c515 100644 --- a/src/rpc/methods/chain.rs +++ b/src/rpc/methods/chain.rs @@ -10,6 +10,7 @@ use crate::blocks::RawBlockHeader; use crate::blocks::{Block, CachingBlockHeader, Tipset, TipsetKey}; use crate::chain::index::ResolveNullTipset; use crate::chain::{ChainStore, ExportOptions, FilecoinSnapshotVersion, HeadChange}; +use crate::chain_sync::get_full_tipset; use crate::cid_collections::CidHashSet; use crate::ipld::DfsIter; use crate::ipld::{CHAIN_EXPORT_STATUS, cancel_export, end_export, start_export}; @@ -281,7 +282,7 @@ impl RpcMethod<1> for ChainGetParentMessages { type Ok = Vec; async fn handle( - ctx: Ctx, + ctx: Ctx, (block_cid,): Self::Params, ) -> Result { let store = ctx.store(); @@ -292,7 +293,7 @@ impl RpcMethod<1> for ChainGetParentMessages { Ok(vec![]) } else { let parent_tipset = Tipset::load_required(store, &block_header.parents)?; - load_api_messages_from_tipset(store, &parent_tipset) + load_api_messages_from_tipset(&ctx, parent_tipset.key()).await } } } @@ -355,13 +356,13 @@ impl RpcMethod<1> for ChainGetMessagesInTipset { type Ok = Vec; async fn handle( - ctx: Ctx, + ctx: Ctx, (ApiTipsetKey(tipset_key),): Self::Params, ) -> Result { let tipset = ctx .chain_store() .load_required_tipset_or_heaviest(&tipset_key)?; - load_api_messages_from_tipset(ctx.store(), &tipset) + load_api_messages_from_tipset(&ctx, tipset.key()).await } } @@ -1301,13 +1302,17 @@ pub(crate) fn chain_notify( receiver } -fn load_api_messages_from_tipset( - store: &impl Blockstore, - tipset: &Tipset, +async fn load_api_messages_from_tipset( + ctx: &crate::rpc::RPCState, + tipset_keys: &TipsetKey, ) -> Result, ServerError> { - let full_tipset = tipset - .fill_from_blockstore(store) - .context("Failed to load full tipset")?; + let full_tipset = get_full_tipset( + &ctx.sync_network_context, + ctx.chain_store(), + None, + tipset_keys, + ) + .await?; let blocks = full_tipset.into_blocks(); let mut messages = vec![]; let mut seen = CidHashSet::default(); From 991411a26549f6c25a249ed6f4b75bcefaf740d3 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 14 Jan 2026 21:14:14 +0800 Subject: [PATCH 2/5] opt-in with FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK --- docs/dictionary.txt | 1 + docs/docs/users/reference/env_variables.md | 1 + src/rpc/methods/chain.rs | 23 ++++++++++++++-------- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/dictionary.txt b/docs/dictionary.txt index 5ef0e82e4db7..ed91172e69a4 100644 --- a/docs/dictionary.txt +++ b/docs/dictionary.txt @@ -1,6 +1,7 @@ 2k APIs backend +backfill backport benchmarking blockstore diff --git a/docs/docs/users/reference/env_variables.md b/docs/docs/users/reference/env_variables.md index 99017d6e73f2..c488020f2c48 100644 --- a/docs/docs/users/reference/env_variables.md +++ b/docs/docs/users/reference/env_variables.md @@ -53,6 +53,7 @@ process. | `FOREST_ZSTD_FRAME_CACHE_DEFAULT_MAX_SIZE` | positive integer | 268435456 | 536870912 | The default zstd frame cache max size in bytes | | `FOREST_JWT_DISABLE_EXP_VALIDATION` | 1 or true | empty | 1 | Whether or not to disable JWT expiration validation | | `FOREST_ETH_BLOCK_CACHE_SIZE` | positive integer | 500 | 1 | The size of Eth block cache | +| `FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK` | 1 or true | false | 1 | Whether or not to backfill full tipsets from the p2p network | ### `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT` diff --git a/src/rpc/methods/chain.rs b/src/rpc/methods/chain.rs index 4d98af65c515..17da9241e0fd 100644 --- a/src/rpc/methods/chain.rs +++ b/src/rpc/methods/chain.rs @@ -10,7 +10,7 @@ use crate::blocks::RawBlockHeader; use crate::blocks::{Block, CachingBlockHeader, Tipset, TipsetKey}; use crate::chain::index::ResolveNullTipset; use crate::chain::{ChainStore, ExportOptions, FilecoinSnapshotVersion, HeadChange}; -use crate::chain_sync::get_full_tipset; +use crate::chain_sync::{get_full_tipset, load_full_tipset}; use crate::cid_collections::CidHashSet; use crate::ipld::DfsIter; use crate::ipld::{CHAIN_EXPORT_STATUS, cancel_export, end_export, start_export}; @@ -28,6 +28,7 @@ use crate::shim::executor::Receipt; use crate::shim::message::Message; use crate::utils::db::CborStoreExt as _; use crate::utils::io::VoidAsyncWriter; +use crate::utils::misc::env::is_env_truthy; use anyhow::{Context as _, Result}; use cid::Cid; use fvm_ipld_blockstore::Blockstore; @@ -1306,13 +1307,19 @@ async fn load_api_messages_from_tipset( ctx: &crate::rpc::RPCState, tipset_keys: &TipsetKey, ) -> Result, ServerError> { - let full_tipset = get_full_tipset( - &ctx.sync_network_context, - ctx.chain_store(), - None, - tipset_keys, - ) - .await?; + static SHOULD_BACKFILL: LazyLock = + LazyLock::new(|| is_env_truthy("FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK")); + let full_tipset = if *SHOULD_BACKFILL { + get_full_tipset( + &ctx.sync_network_context, + ctx.chain_store(), + None, + tipset_keys, + ) + .await? + } else { + load_full_tipset(ctx.chain_store(), tipset_keys)? + }; let blocks = full_tipset.into_blocks(); let mut messages = vec![]; let mut seen = CidHashSet::default(); From 9a78ef988cf0cd43f6e38939ad722903e96e949a Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 14 Jan 2026 21:57:52 +0800 Subject: [PATCH 3/5] changelog and warning --- CHANGELOG.md | 2 ++ src/rpc/methods/chain.rs | 3 +++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 651cac711cb1..7077a85daa53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,8 @@ - [#6405](https://github.com/ChainSafe/forest/pull/6405) Enabled `Filecoin.EthGetLogs` for API v2. +- [#6421](https://github.com/ChainSafe/forest/pull/6421) Add an environment variable `FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK` to enable backfilling full tipsets from network in a few RPC metheods. + ### Changed - [#6368](https://github.com/ChainSafe/forest/pull/6368): Migrated build and development tooling from Makefile to `mise`. Contributors should install `mise` and use `mise run` commands instead of `make` commands. diff --git a/src/rpc/methods/chain.rs b/src/rpc/methods/chain.rs index 17da9241e0fd..d9f1a5f5814f 100644 --- a/src/rpc/methods/chain.rs +++ b/src/rpc/methods/chain.rs @@ -1310,6 +1310,9 @@ async fn load_api_messages_from_tipset( static SHOULD_BACKFILL: LazyLock = LazyLock::new(|| is_env_truthy("FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK")); let full_tipset = if *SHOULD_BACKFILL { + tracing::warn!( + "Full tipset backfilling from network is enabled via FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK, excessive disk and bandwidth usage is expected." + ); get_full_tipset( &ctx.sync_network_context, ctx.chain_store(), From 4705b462c0a26fcce8f780b4eea6b46b63f19fc3 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 14 Jan 2026 22:04:06 +0800 Subject: [PATCH 4/5] typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7077a85daa53..67dcb37a1a1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,7 @@ - [#6405](https://github.com/ChainSafe/forest/pull/6405) Enabled `Filecoin.EthGetLogs` for API v2. -- [#6421](https://github.com/ChainSafe/forest/pull/6421) Add an environment variable `FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK` to enable backfilling full tipsets from network in a few RPC metheods. +- [#6421](https://github.com/ChainSafe/forest/pull/6421) Add an environment variable `FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK` to enable backfilling full tipsets from network in a few RPC methods. ### Changed From 49559dbc5e95c378f94a0fa67d780b86c443d489 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 15 Jan 2026 17:54:57 +0800 Subject: [PATCH 5/5] print warning only once --- src/rpc/methods/chain.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/rpc/methods/chain.rs b/src/rpc/methods/chain.rs index d9f1a5f5814f..64f1223ea3e0 100644 --- a/src/rpc/methods/chain.rs +++ b/src/rpc/methods/chain.rs @@ -1307,12 +1307,16 @@ async fn load_api_messages_from_tipset( ctx: &crate::rpc::RPCState, tipset_keys: &TipsetKey, ) -> Result, ServerError> { - static SHOULD_BACKFILL: LazyLock = - LazyLock::new(|| is_env_truthy("FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK")); + static SHOULD_BACKFILL: LazyLock = LazyLock::new(|| { + let enabled = is_env_truthy("FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK"); + if enabled { + tracing::warn!( + "Full tipset backfilling from network is enabled via FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK, excessive disk and bandwidth usage is expected." + ); + } + enabled + }); let full_tipset = if *SHOULD_BACKFILL { - tracing::warn!( - "Full tipset backfilling from network is enabled via FOREST_RPC_BACKFILL_FULL_TIPSET_FROM_NETWORK, excessive disk and bandwidth usage is expected." - ); get_full_tipset( &ctx.sync_network_context, ctx.chain_store(),