Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/chain/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use multihash_derive::MultihashDigest as _;
use nunny::Vec as NonEmpty;
use sha2::digest::{self, Digest};
use std::io::{Read, Seek, SeekFrom};
use std::time::Instant;
use tokio::io::{AsyncWrite, AsyncWriteExt, BufWriter};

pub struct ExportOptions<S> {
Expand Down Expand Up @@ -146,6 +147,13 @@ async fn export_to_forest_car<D: Digest, S: CidHashSetLike + Send + Sync + 'stat
anyhow::bail!("message receipts must be included when events are included");
}

let start = Instant::now();
tracing::info!(
"Exporting snapshot, epoch={}, depth={lookup_depth}, prefix_frames={}",
tipset.epoch(),
prefix_data_frames.as_ref().map(|v| v.len()).unwrap_or(0)
);

let stateroot_lookup_limit = tipset.epoch() - lookup_depth;

// Wrap writer in optional checksum calculator
Expand Down Expand Up @@ -182,5 +190,10 @@ async fn export_to_forest_car<D: Digest, S: CidHashSetLike + Send + Sync + 'stat

let digest = writer.finalize().map_err(|e| Error::Other(e.to_string()))?;

tracing::info!(
"Exported snapshot, took {}",
humantime::format_duration(start.elapsed())
);

Ok(digest)
}
9 changes: 7 additions & 2 deletions src/db/car/forest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -333,21 +333,26 @@ impl Encoder {

// Write seekable zstd and collect a mapping of CIDs to frame_offset+data_offset.
let mut builder = index::Builder::new();
let mut n_frames = 0;
while let Some((cids, zstd_frame)) = stream.try_next().await? {
builder.extend(cids.into_iter().map(|cid| (cid, offset as u64)));
sink.write_all(&zstd_frame).await?;
offset += zstd_frame.len()
offset += zstd_frame.len();
n_frames += 1;
}

tracing::info!("Finished writing {n_frames} zstd CAR frames");

// Create index
let writer = builder.into_writer();
writer.write_zstd_skip_frames_into(&mut sink).await?;

tracing::info!("Finished writing zstd CAR index frames");
// Write ForestCAR.zst footer, it's a valid ZSTD skip-frame
let footer = ForestCarFooter {
index: offset as u64 + ZSTD_SKIP_FRAME_LEN,
};
sink.write_all(&footer.to_le_bytes()).await?;
tracing::info!("Finished writing zstd CAR footer frame");
Ok(())
}

Expand Down
Loading