Skip to content

feat: storing block hashes along with the headers in segments #351

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xdustinface merged 2 commits into from
Jan 13, 2026
Merged

feat: storing block hashes along with the headers in segments #351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
23faa6f
index header no longer persisted in a different file
ZocoLini Jan 12, 2026
f836f9d
CachedHeader renamed and public methods modified
ZocoLini Jan 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 26 additions & 29 deletions dash-spv/src/storage/blocks.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,9 @@ use dashcore::BlockHash;
use tokio::sync::RwLock;

use crate::error::StorageResult;
use crate::storage::io::atomic_write;
use crate::storage::segments::SegmentCache;
use crate::storage::PersistentStorage;
use crate::StorageError;
use crate::types::HashedBlockHeader;

#[async_trait]
pub trait BlockHeaderStorage {
Expand Down Expand Up @@ -60,13 +59,12 @@ pub trait BlockHeaderStorage {
}

pub struct PersistentBlockHeaderStorage {
block_headers: RwLock<SegmentCache<BlockHeader>>,
block_headers: RwLock<SegmentCache<HashedBlockHeader>>,
header_hash_index: HashMap<BlockHash, u32>,
}

impl PersistentBlockHeaderStorage {
const FOLDER_NAME: &str = "block_headers";
const INDEX_FILE_NAME: &str = "index.dat";
}

#[async_trait]
Expand All @@ -75,24 +73,19 @@ impl PersistentStorage for PersistentBlockHeaderStorage {
let storage_path = storage_path.into();
let segments_folder = storage_path.join(Self::FOLDER_NAME);

let index_path = segments_folder.join(Self::INDEX_FILE_NAME);
let mut block_headers: SegmentCache<HashedBlockHeader> =
SegmentCache::load_or_new(&segments_folder).await?;

let mut block_headers = SegmentCache::load_or_new(&segments_folder).await?;
let mut header_hash_index = HashMap::new();

let header_hash_index = match tokio::fs::read(&index_path)
.await
.ok()
.and_then(|content| bincode::deserialize(&content).ok())
if let (Some(start), Some(end)) = (block_headers.start_height(), block_headers.tip_height())
{
Some(index) => index,
_ => {
if segments_folder.exists() {
block_headers.build_block_index_from_segments().await?
} else {
HashMap::new()
}
let headers = block_headers.get_items(start..end + 1).await?;
for (i, header) in headers.iter().enumerate() {
let height = start + i as u32;
header_hash_index.insert(*header.hash(), height);
}
};
}

Ok(Self {
block_headers: RwLock::new(block_headers),
Expand All @@ -102,16 +95,12 @@ impl PersistentStorage for PersistentBlockHeaderStorage {

async fn persist(&mut self, storage_path: impl Into<PathBuf> + Send) -> StorageResult<()> {
let block_headers_folder = storage_path.into().join(Self::FOLDER_NAME);
let index_path = block_headers_folder.join(Self::INDEX_FILE_NAME);

tokio::fs::create_dir_all(&block_headers_folder).await?;

self.block_headers.write().await.persist(&block_headers_folder).await;

let data = bincode::serialize(&self.header_hash_index)
.map_err(|e| StorageError::WriteFailed(format!("Failed to serialize index: {}", e)))?;

atomic_write(&index_path, &data).await
Ok(())
}
}

Expand All @@ -128,21 +117,29 @@ impl BlockHeaderStorage for PersistentBlockHeaderStorage {
height: u32,
) -> StorageResult<()> {
let mut height = height;
let headers =
headers.iter().map(HashedBlockHeader::from).collect::<Vec<HashedBlockHeader>>();

let hashes = headers.iter().map(|header| header.block_hash()).collect::<Vec<_>>();

self.block_headers.write().await.store_items_at_height(headers, height).await?;
self.block_headers.write().await.store_items_at_height(&headers, height).await?;

for hash in hashes {
self.header_hash_index.insert(hash, height);
for header in headers {
self.header_hash_index.insert(*header.hash(), height);
height += 1;
}

Ok(())
}

async fn load_headers(&self, range: Range<u32>) -> StorageResult<Vec<BlockHeader>> {
self.block_headers.write().await.get_items(range).await
Ok(self
.block_headers
.write()
.await
.get_items(range)
.await?
.into_iter()
.map(|cached| *cached.header())
.collect())
}

async fn get_tip_height(&self) -> Option<u32> {
Expand Down
54 changes: 8 additions & 46 deletions dash-spv/src/storage/segments.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ use dashcore::{
};
use dashcore_hashes::Hash;

use crate::{error::StorageResult, storage::io::atomic_write, StorageError};
use crate::{
error::StorageResult, storage::io::atomic_write, types::HashedBlockHeader, StorageError,
};

pub trait Persistable: Sized + Encodable + Decodable + PartialEq + Clone {
const SEGMENT_PREFIX: &'static str = "segment";
Expand All @@ -36,16 +38,18 @@ impl Persistable for Vec<u8> {
}
}

impl Persistable for BlockHeader {
impl Persistable for HashedBlockHeader {
fn sentinel() -> Self {
Self {
let header = BlockHeader {
version: Version::from_consensus(i32::MAX), // Invalid version
prev_blockhash: BlockHash::from_byte_array([0xFF; 32]), // All 0xFF pattern
merkle_root: dashcore::hashes::sha256d::Hash::from_byte_array([0xFF; 32]).into(),
time: u32::MAX, // Far future timestamp
bits: CompactTarget::from_consensus(0xFFFFFFFF), // Invalid difficulty
nonce: u32::MAX,
}
};

Self::from(header)
}
}

Expand All @@ -65,48 +69,6 @@ pub struct SegmentCache<I: Persistable> {
segments_dir: PathBuf,
}

impl SegmentCache<BlockHeader> {
pub async fn build_block_index_from_segments(
&mut self,
) -> StorageResult<HashMap<BlockHash, u32>> {
let entries = fs::read_dir(&self.segments_dir)?;

let mut block_index = HashMap::new();

for entry in entries.flatten() {
let name = match entry.file_name().into_string() {
Ok(s) => s,
Err(_) => continue,
};

if !name.starts_with(BlockHeader::SEGMENT_PREFIX) {
continue;
}

if !name.ends_with(&format!(".{}", BlockHeader::DATA_FILE_EXTENSION)) {
continue;
}

let segment_id = match name[8..12].parse::<u32>() {
Ok(id) => id,
Err(_) => continue,
};

let mut block_height = Self::segment_id_to_start_height(segment_id);

let segment = self.get_segment(&segment_id).await?;

for item in segment.items.iter() {
block_index.insert(item.block_hash(), block_height);

block_height += 1;
}
}

Ok(block_index)
}
}

impl<I: Persistable> SegmentCache<I> {
const MAX_ACTIVE_SEGMENTS: usize = 10;

Expand Down
23 changes: 11 additions & 12 deletions dash-spv/src/sync/headers/manager.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use crate::network::NetworkManager;
use crate::storage::StorageManager;
use crate::sync::headers::validate_headers;
use crate::sync::headers2::Headers2StateManager;
use crate::types::{CachedHeader, ChainState};
use crate::types::{ChainState, HashedBlockHeader};
use crate::ValidationMode;
use std::sync::Arc;
use tokio::sync::RwLock;
Expand Down Expand Up @@ -133,8 +133,7 @@ impl<S: StorageManager, N: NetworkManager> HeaderSyncManager<S, N> {

// Wrap headers in CachedHeader to avoid redundant X11 hashing
// This prevents recomputing hashes during validation, logging, and storage
let cached_headers: Vec<CachedHeader> =
headers.iter().map(|h| CachedHeader::new(*h)).collect();
let cached_headers: Vec<_> = headers.iter().map(HashedBlockHeader::from).collect();

// Step 2: Validate Batch
let first_cached = &cached_headers[0];
Expand All @@ -154,10 +153,10 @@ impl<S: StorageManager, N: NetworkManager> HeaderSyncManager<S, N> {

// Check if the first header connects to our tip
// Cache tip hash to avoid recomputing it
let tip_cached = CachedHeader::new(tip);
let tip_hash = tip_cached.block_hash();
let tip_cached = HashedBlockHeader::from(tip);
let tip_hash = tip_cached.hash();

if first_header.prev_blockhash != tip_hash {
if first_header.prev_blockhash != *tip_hash {
tracing::warn!(
"Received header batch that does not connect to our tip. Expected prev_hash: {}, got: {}. Dropping message.",
tip_hash,
Expand Down Expand Up @@ -201,8 +200,8 @@ impl<S: StorageManager, N: NetworkManager> HeaderSyncManager<S, N> {
if !cached_headers.is_empty() {
let last_cached = cached_headers.last().unwrap();
// Use cached hashes to avoid redundant X11 computation
let first_hash = first_cached.block_hash();
let last_hash = last_cached.block_hash();
let first_hash = first_cached.hash();
let last_hash = last_cached.hash();
tracing::debug!(
"Received headers batch: first.prev_hash={}, first.hash={}, last.hash={}, count={}",
first_header.prev_blockhash,
Expand All @@ -220,8 +219,8 @@ impl<S: StorageManager, N: NetworkManager> HeaderSyncManager<S, N> {

if self.reorg_config.enforce_checkpoints {
// Use cached hash to avoid redundant X11 computation in loop
let header_hash = cached_header.block_hash();
if !self.checkpoint_manager.validate_block(prospective_height, &header_hash) {
let header_hash = cached_header.hash();
if !self.checkpoint_manager.validate_block(prospective_height, header_hash) {
return Err(SyncError::Validation(format!(
"Block at height {} does not match checkpoint",
prospective_height
Expand Down Expand Up @@ -259,8 +258,8 @@ impl<S: StorageManager, N: NetworkManager> HeaderSyncManager<S, N> {
// During sync mode - request next batch
// Use the last cached header's hash to avoid redundant X11 computation
if let Some(last_cached) = cached_headers.last() {
let hash = last_cached.block_hash();
self.request_headers(network, Some(hash), storage).await?;
let hash = last_cached.hash();
self.request_headers(network, Some(*hash), storage).await?;
}
}

Expand Down
36 changes: 18 additions & 18 deletions dash-spv/src/sync/headers/validation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,32 @@ use rayon::prelude::*;
use std::time::Instant;

use crate::error::{ValidationError, ValidationResult};
use crate::types::CachedHeader;
use crate::types::HashedBlockHeader;

/// Validate a chain of headers.
pub fn validate_headers(headers: &[CachedHeader]) -> ValidationResult<()> {
pub fn validate_headers(hashed_headers: &[HashedBlockHeader]) -> ValidationResult<()> {
let start = Instant::now();

// Check PoW of i and continuity of i-1 to i in parallel
headers.par_iter().enumerate().try_for_each(|(i, header)| {
hashed_headers.par_iter().enumerate().try_for_each(|(i, header)| {
// For the first header, skip chain continuity check since we don't have i-1 here
if i > 0 && header.prev_blockhash != headers[i - 1].block_hash() {
if i > 0 && header.header().prev_blockhash != *hashed_headers[i - 1].hash() {
return Err(ValidationError::InvalidHeaderChain(format!(
"Header {:?} does not connect to {:?}",
headers[i - 1],
hashed_headers[i - 1],
header
)));
}
// Check if PoW target is met
if !header.target().is_met_by(header.block_hash()) {
if !header.header().target().is_met_by(*header.hash()) {
return Err(ValidationError::InvalidProofOfWork);
}
Ok(())
})?;

tracing::trace!(
"Header chain validation passed for {} headers, duration: {:?}",
headers.len(),
hashed_headers.len(),
start.elapsed(),
);

Expand All @@ -40,7 +40,7 @@ pub fn validate_headers(headers: &[CachedHeader]) -> ValidationResult<()> {
mod tests {
use super::validate_headers;
use crate::error::ValidationError;
use crate::types::CachedHeader;
use crate::types::HashedBlockHeader;
use dashcore::{
block::{Header as BlockHeader, Version},
blockdata::constants::genesis_block,
Expand All @@ -51,8 +51,8 @@ mod tests {
// Very easy target to pass PoW checks for continuity tests
const MAX_TARGET: u32 = 0x2100ffff;

fn create_test_header(prev_hash: dashcore::BlockHash, nonce: u32) -> CachedHeader {
CachedHeader::new(BlockHeader {
fn create_test_header(prev_hash: dashcore::BlockHash, nonce: u32) -> HashedBlockHeader {
HashedBlockHeader::from(BlockHeader {
version: Version::from_consensus(1),
prev_blockhash: prev_hash,
merkle_root: dashcore::TxMerkleNode::all_zeros(),
Expand Down Expand Up @@ -80,7 +80,7 @@ mod tests {

for i in 0..10 {
let header = create_test_header(prev_hash, i);
prev_hash = header.block_hash();
prev_hash = *header.hash();
headers.push(header);
}

Expand All @@ -90,7 +90,7 @@ mod tests {
#[test]
fn test_broken_chain() {
let header1 = create_test_header(dashcore::BlockHash::all_zeros(), 0);
let header2 = create_test_header(header1.block_hash(), 1);
let header2 = create_test_header(*header1.hash(), 1);
// header3 doesn't connect to header2
let header3 = create_test_header(dashcore::BlockHash::all_zeros(), 2);

Expand All @@ -100,7 +100,7 @@ mod tests {

#[test]
fn test_invalid_pow() {
let header = CachedHeader::new(BlockHeader {
let header = HashedBlockHeader::from(BlockHeader {
version: Version::from_consensus(1),
prev_blockhash: dashcore::BlockHash::all_zeros(),
merkle_root: dashcore::TxMerkleNode::all_zeros(),
Expand All @@ -116,7 +116,7 @@ mod tests {
#[test]
fn test_genesis_blocks() {
for network in [Network::Dash, Network::Testnet, Network::Regtest] {
let genesis = CachedHeader::new(genesis_block(network).header);
let genesis = HashedBlockHeader::from(genesis_block(network).header);
assert!(
validate_headers(&[genesis]).is_ok(),
"Genesis block for {:?} should validate",
Expand All @@ -128,19 +128,19 @@ mod tests {
#[test]
fn test_invalid_pow_mid_chain() {
let header1 = create_test_header(dashcore::BlockHash::all_zeros(), 0);
let header2 = create_test_header(header1.block_hash(), 1);
let header2 = create_test_header(*header1.hash(), 1);

// Header 3 has valid continuity but impossible PoW target
let header3 = CachedHeader::new(BlockHeader {
let header3 = HashedBlockHeader::from(BlockHeader {
version: Version::from_consensus(1),
prev_blockhash: header2.block_hash(),
prev_blockhash: *header2.hash(),
merkle_root: dashcore::TxMerkleNode::all_zeros(),
time: 0,
bits: CompactTarget::from_consensus(0x1d00ffff), // Hard target
nonce: 0,
});

let header4 = create_test_header(header3.block_hash(), 3);
let header4 = create_test_header(*header3.hash(), 3);

let result = validate_headers(&[header1, header2, header3, header4]);
assert!(matches!(result, Err(ValidationError::InvalidProofOfWork)));
Expand Down
Loading
Loading