diff --git a/barretenberg/cpp/src/barretenberg/benchmark/chonk_bench/chonk.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/chonk_bench/chonk.bench.cpp index 025fbeb4ab8b..a05c780f3c13 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/chonk_bench/chonk.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/chonk_bench/chonk.bench.cpp @@ -124,17 +124,23 @@ BENCHMARK_REGISTER_F(ChonkBench, ProofDecompress)->Unit(benchmark::kMillisecond) BENCHMARK_REGISTER_F(ChonkBench, VerifyIndividual)->Unit(benchmark::kMillisecond)->Arg(1)->Arg(2)->Arg(4)->Arg(8); /** - * @brief Benchmark BN254 G1 point decompression (used by SRS compressed download) + * @brief Benchmark BN254 G1 point decompression (used during compressed CRS download) */ void bn254_point_decompression(benchmark::State& state) { constexpr size_t NUM_POINTS = 1 << 17; // 131072 — typical circuit size - // Read compressed points from disk (32 bytes each, big-endian uint256_t) - auto compressed_buf = read_file(bb::srs::bb_crs_path() / "bn254_g1_compressed.dat", NUM_POINTS * sizeof(uint256_t)); + // Read uncompressed points from disk and compress them for benchmark input. + // Compression: store x-coordinate as uint256_t, set bit 255 if y is odd. + auto uncompressed_buf = read_file(bb::srs::bb_crs_path() / "bn254_g1.dat", NUM_POINTS * sizeof(g1::affine_element)); std::vector compressed(NUM_POINTS); for (size_t i = 0; i < NUM_POINTS; ++i) { - compressed[i] = from_buffer(compressed_buf, i * sizeof(uint256_t)); + auto point = from_buffer(uncompressed_buf, i * sizeof(g1::affine_element)); + uint256_t x(point.x); + if (uint256_t(point.y).get_bit(0)) { + x.data[3] |= bb::group_elements::UINT256_TOP_LIMB_MSB; + } + compressed[i] = x; } for (auto _ : state) { diff --git a/barretenberg/cpp/src/barretenberg/srs/factories/crs_factory.test.cpp b/barretenberg/cpp/src/barretenberg/srs/factories/crs_factory.test.cpp index 01fe026da882..8963c7eac08f 100644 --- a/barretenberg/cpp/src/barretenberg/srs/factories/crs_factory.test.cpp +++ b/barretenberg/cpp/src/barretenberg/srs/factories/crs_factory.test.cpp @@ -25,12 +25,11 @@ void check_bn254_consistency(const fs::path& crs_download_path, size_t num_point { NativeBn254CrsFactory file_crs(crs_download_path, allow_download); - // read compressed G1 and decompress - auto g1_compressed = read_file(bb::srs::bb_crs_path() / "bn254_g1_compressed.dat", num_points * sizeof(uint256_t)); + // Read uncompressed G1 points (64 bytes each) from the on-disk cache + auto g1_data = read_file(bb::srs::bb_crs_path() / "bn254_g1.dat", num_points * sizeof(g1::affine_element)); std::vector g1_points(num_points); for (size_t i = 0; i < num_points; ++i) { - auto c = from_buffer(g1_compressed, i * sizeof(uint256_t)); - g1_points[i] = g1::affine_element::from_compressed(c); + g1_points[i] = from_buffer(g1_data, i * sizeof(g1::affine_element)); } // read G2 @@ -126,23 +125,3 @@ TEST(CrsFactory, DISABLED_Bn254Fallback) fs::remove_all(temp_crs_path); } - -TEST(CrsFactory, Bn254CompressedChunkHashFirstChunk) -{ - // Verify that the first 4MB chunk of the compressed CRS matches the embedded hash - auto data = read_file(bb::srs::bb_crs_path() / "bn254_g1_compressed.dat", bb::srs::SRS_CHUNK_SIZE_BYTES); - auto chunk = std::span(data.data(), data.size()); - auto hash = bb::crypto::sha256(chunk); - EXPECT_EQ(hash, bb::srs::BN254_G1_CHUNK_HASHES[0]); -} - -TEST(CrsFactory, Bn254CompressedChunkHashCorruptionDetected) -{ - // Verify that corrupted data fails chunk hash verification - auto data = read_file(bb::srs::bb_crs_path() / "bn254_g1_compressed.dat", bb::srs::SRS_CHUNK_SIZE_BYTES); - - data[bb::srs::SRS_CHUNK_SIZE_BYTES / 2] ^= 0xFF; - auto chunk = std::span(data.data(), data.size()); - auto hash = bb::crypto::sha256(chunk); - EXPECT_NE(hash, bb::srs::BN254_G1_CHUNK_HASHES[0]); -} diff --git a/barretenberg/cpp/src/barretenberg/srs/factories/get_bn254_crs.cpp b/barretenberg/cpp/src/barretenberg/srs/factories/get_bn254_crs.cpp index db0c179ab55f..22644347fa37 100644 --- a/barretenberg/cpp/src/barretenberg/srs/factories/get_bn254_crs.cpp +++ b/barretenberg/cpp/src/barretenberg/srs/factories/get_bn254_crs.cpp @@ -9,11 +9,12 @@ #include "bn254_crs_data.hpp" #include "bn254_g1_chunk_hashes.hpp" #include "http_download.hpp" +#include #include #include namespace { -// Primary CRS URL (Cloudflare R2) +// Primary CRS URL (Cloudflare R2) — compressed format (32 bytes/point) constexpr const char* CRS_PRIMARY_URL = "http://crs.aztec-cdn.foundation/g1_compressed.dat"; // Fallback CRS URL (AWS S3) constexpr const char* CRS_FALLBACK_URL = "http://crs.aztec-labs.com/g1_compressed.dat"; @@ -90,23 +91,15 @@ void verify_bn254_crs_integrity(const std::vector& data) } /** - * @brief Decompress a buffer of compressed G1 points (32 bytes each) into affine elements. + * @brief Download compressed CRS data, verify integrity, decompress, and serialize to uncompressed format. + * + * @details Downloads 32 bytes/point (compressed), verifies SHA-256 chunk hashes, + * then decompresses to 64 bytes/point (uncompressed affine x,y) for on-disk storage. + * This gives us 50% bandwidth savings while keeping the on-disk format fast to load. */ -std::vector decompress_g1_points(const std::vector& data, size_t num_points) -{ - std::vector points(num_points); - bb::parallel_for([&](bb::ThreadChunk chunk) { - for (auto i : chunk.range(num_points)) { - uint256_t compressed = from_buffer(data, i * COMPRESSED_POINT_SIZE); - points[i] = bb::g1::affine_element::from_compressed(compressed); - } - }); - return points; -} - -std::vector download_bn254_g1_data(size_t num_points, - const std::string& primary_url, - const std::string& fallback_url) +std::vector download_and_decompress_bn254_g1_data(size_t num_points, + const std::string& primary_url, + const std::string& fallback_url) { // Round up to chunk boundary so every downloaded byte is hash-verified size_t download_points = round_up_to_chunk_boundary(num_points); @@ -115,41 +108,62 @@ std::vector download_bn254_g1_data(size_t num_points, // Try primary URL first, with fallback on failure. // Note: WASM is compiled with -fno-exceptions, so try/catch is not available. // In practice, WASM never calls this function - it initializes CRS via srs_init_srs from JavaScript. - std::vector data; + std::vector compressed_data; #ifndef __wasm__ try { - data = bb::srs::http_download(primary_url, 0, g1_end); + compressed_data = bb::srs::http_download(primary_url, 0, g1_end); } catch (const std::exception& e) { vinfo("Primary CRS download failed: ", e.what(), ". Trying fallback..."); - data = bb::srs::http_download(fallback_url, 0, g1_end); + compressed_data = bb::srs::http_download(fallback_url, 0, g1_end); } #else // WASM fallback: just try primary (will abort on failure) - data = bb::srs::http_download(primary_url, 0, g1_end); + compressed_data = bb::srs::http_download(primary_url, 0, g1_end); static_cast(fallback_url); #endif - if (data.size() < COMPRESSED_POINT_SIZE) { + if (compressed_data.size() < COMPRESSED_POINT_SIZE) { throw_or_abort("Downloaded g1 data is too small"); } // Quick sanity check: verify the first two G1 points match expected values - auto first = from_buffer(data, 0); + auto first = from_buffer(compressed_data, 0); if (first != bb::srs::BN254_G1_FIRST_ELEMENT_COMPRESSED) { throw_or_abort("Downloaded BN254 G1 CRS first element does not match expected point."); } - if (data.size() >= 2 * COMPRESSED_POINT_SIZE) { - auto second = from_buffer(data, COMPRESSED_POINT_SIZE); + if (compressed_data.size() >= 2 * COMPRESSED_POINT_SIZE) { + auto second = from_buffer(compressed_data, COMPRESSED_POINT_SIZE); if (second != bb::srs::BN254_G1_SECOND_ELEMENT_COMPRESSED) { throw_or_abort("Downloaded BN254 G1 CRS second element does not match expected point."); } } // Full integrity verification: SHA-256 chunk hashes in parallel - verify_bn254_crs_integrity(data); + verify_bn254_crs_integrity(compressed_data); + + // Decompress to affine elements and serialize to uncompressed format (64 bytes/point) + size_t actual_points = compressed_data.size() / COMPRESSED_POINT_SIZE; + std::vector points(actual_points); + bb::parallel_for([&](bb::ThreadChunk chunk) { + for (auto i : chunk.range(actual_points)) { + uint256_t c = from_buffer(compressed_data, i * COMPRESSED_POINT_SIZE); + points[i] = bb::g1::affine_element::from_compressed(c); + } + }); + + // Serialize to uncompressed format for on-disk storage + std::vector uncompressed_data(actual_points * sizeof(bb::g1::affine_element)); + bb::parallel_for([&](bb::ThreadChunk chunk) { + for (auto i : chunk.range(actual_points)) { + auto buf = to_buffer(points[i]); + std::copy(buf.begin(), + buf.end(), + uncompressed_data.begin() + static_cast(i * sizeof(bb::g1::affine_element))); + } + }); - return data; + return uncompressed_data; } } // namespace @@ -166,41 +180,81 @@ std::vector get_bn254_g1_data(const std::filesystem::path& p BB_BENCH_NAME("get_bn254_g1_data"); std::filesystem::create_directories(path); + auto g1_path = path / "bn254_g1.dat"; auto compressed_path = path / "bn254_g1_compressed.dat"; auto lock_path = path / "crs.lock"; // Acquire exclusive lock to prevent simultaneous downloads FileLockGuard lock(lock_path.string()); - size_t compressed_points = get_file_size(compressed_path) / COMPRESSED_POINT_SIZE; - if (compressed_points >= num_points) { - vinfo("using cached bn254 crs with ", std::to_string(compressed_points), " points at ", compressed_path); + auto deserialize_points = [](const std::vector& data, size_t n) { + auto points = std::vector(n); + parallel_for([&](const ThreadChunk& tc) { + for (size_t i : tc.range(n)) { + points[i] = from_buffer(data, i * sizeof(g1::affine_element)); + } + }); + return points; + }; + + auto decompress_points = [](const std::vector& data, size_t n) { + auto points = std::vector(n); + parallel_for([&](const ThreadChunk& tc) { + for (size_t i : tc.range(n)) { + uint256_t c = from_buffer(data, i * COMPRESSED_POINT_SIZE); + points[i] = g1::affine_element::from_compressed(c); + } + }); + return points; + }; + + // Check uncompressed file first (bn254_g1.dat, 64 bytes/point) + size_t g1_file_points = get_file_size(g1_path) / sizeof(g1::affine_element); + if (g1_file_points >= num_points) { + vinfo("using cached bn254 crs with ", std::to_string(g1_file_points), " points at ", g1_path); + auto data = read_file(g1_path, num_points * sizeof(g1::affine_element)); + return deserialize_points(data, num_points); + } + + // Fall back to compressed file (bn254_g1_compressed.dat, 32 bytes/point) + size_t compressed_file_points = get_file_size(compressed_path) / COMPRESSED_POINT_SIZE; + if (compressed_file_points >= num_points) { + vinfo("using cached compressed bn254 crs with ", + std::to_string(compressed_file_points), + " points at ", + compressed_path); auto data = read_file(compressed_path, num_points * COMPRESSED_POINT_SIZE); - return decompress_g1_points(data, num_points); + return decompress_points(data, num_points); } - if (!allow_download && compressed_points == 0) { - throw_or_abort("bn254 g1 compressed data not found at " + compressed_path.string() + + if (!allow_download && g1_file_points == 0 && compressed_file_points == 0) { + throw_or_abort("bn254 g1 data not found at " + g1_path.string() + " and bb does not automatically download in this context." + " Run barretenberg/crs/bootstrap.sh to download."); } else if (!allow_download) { throw_or_abort(format("bn254 g1 data had ", - compressed_points, + std::max(g1_file_points, compressed_file_points), " points and ", num_points, " were requested but download not allowed in this context")); } // Double-check after acquiring lock (another process may have downloaded while we waited) - compressed_points = get_file_size(compressed_path) / COMPRESSED_POINT_SIZE; - if (compressed_points >= num_points) { + g1_file_points = get_file_size(g1_path) / sizeof(g1::affine_element); + if (g1_file_points >= num_points) { + auto data = read_file(g1_path, num_points * sizeof(g1::affine_element)); + return deserialize_points(data, num_points); + } + compressed_file_points = get_file_size(compressed_path) / COMPRESSED_POINT_SIZE; + if (compressed_file_points >= num_points) { auto data = read_file(compressed_path, num_points * COMPRESSED_POINT_SIZE); - return decompress_g1_points(data, num_points); + return decompress_points(data, num_points); } + // Download compressed, verify, decompress, and store uncompressed on disk vinfo("downloading bn254 crs..."); - auto data = download_bn254_g1_data(num_points, primary_url, fallback_url); - write_file(compressed_path, data); - return decompress_g1_points(data, num_points); + auto data = download_and_decompress_bn254_g1_data(num_points, primary_url, fallback_url); + write_file(g1_path, data); + return deserialize_points(data, num_points); } // Default overload using production URLs diff --git a/barretenberg/crs/bootstrap.sh b/barretenberg/crs/bootstrap.sh index 1a16fddb0db5..9d16e74d1b18 100755 --- a/barretenberg/crs/bootstrap.sh +++ b/barretenberg/crs/bootstrap.sh @@ -6,7 +6,7 @@ shift || true # To run bb we need a crs. # Download ignition up front to ensure no race conditions at runtime. -# 2^25 points + 1 because the first is the generator, *32 bytes per compressed point, -1 because Range is inclusive. +# 2^25 points + 1 because the first is the generator, *64 bytes per point, -1 because Range is inclusive. # We make the file read only to ensure no test can attempt to grow it any larger. 2^25 is already huge... # TODO: Make bb just download and append/overwrite required range, then it becomes idempotent. @@ -47,14 +47,14 @@ download_with_fallback() { function build { crs_path=$HOME/.bb-crs crs_size=$((2**25+1)) - crs_size_bytes=$((crs_size*32)) - g1=$crs_path/bn254_g1_compressed.dat + crs_size_bytes=$((crs_size*64)) + g1=$crs_path/bn254_g1.dat g2=$crs_path/bn254_g2.dat if [ ! -f "$g1" ] || [ $(stat -c%s "$g1") -lt $crs_size_bytes ]; then - echo "Downloading compressed crs of size: ${crs_size} ($((crs_size_bytes/(1024*1024)))MB)" + echo "Downloading crs of size: ${crs_size} ($((crs_size_bytes/(1024*1024)))MB)" mkdir -p $crs_path - download_with_fallback "$g1" "g1_compressed.dat" "bytes=0-$((crs_size_bytes-1))" - chmod a-w "$g1" + download_with_fallback "$g1" "g1.dat" "bytes=0-$((crs_size_bytes-1))" + chmod a-w $crs_path/bn254_g1.dat fi if [ ! -f "$g2" ]; then download_with_fallback "$g2" "g2.dat" diff --git a/barretenberg/scripts/download_bb_crs.sh b/barretenberg/scripts/download_bb_crs.sh index 56199ed88161..053fca09b6d8 100755 --- a/barretenberg/scripts/download_bb_crs.sh +++ b/barretenberg/scripts/download_bb_crs.sh @@ -3,7 +3,7 @@ set -eu # To run bb we need a crs. # Download ignition up front to ensure no race conditions at runtime. -# 2^25 points + 1 because the first is the generator, *32 bytes per compressed point, -1 because Range is inclusive. +# 2^25 points + 1 because the first is the generator, *64 bytes per point, -1 because Range is inclusive. # We make the file read only to ensure no test can attempt to grow it any larger. 2^25 is already huge... # TODO: Make bb just download and append/overwrite required range, then it becomes idempotent. @@ -43,14 +43,14 @@ download_with_fallback() { crs_path=$HOME/.bb-crs crs_size=$((2**25+1)) -crs_size_bytes=$((crs_size*32)) -g1=$crs_path/bn254_g1_compressed.dat +crs_size_bytes=$((crs_size*64)) +g1=$crs_path/bn254_g1.dat g2=$crs_path/bn254_g2.dat if [ ! -f "$g1" ] || [ $(stat -c%s "$g1") -lt $crs_size_bytes ]; then - echo "Downloading compressed crs of size: ${crs_size} ($((crs_size_bytes/(1024*1024)))MB)" + echo "Downloading crs of size: ${crs_size} ($((crs_size_bytes/(1024*1024)))MB)" mkdir -p $crs_path - download_with_fallback "$g1" "g1_compressed.dat" "bytes=0-$((crs_size_bytes-1))" - chmod a-w "$g1" + download_with_fallback "$g1" "g1.dat" "bytes=0-$((crs_size_bytes-1))" + chmod a-w $crs_path/bn254_g1.dat fi if [ ! -f "$g2" ]; then download_with_fallback "$g2" "g2.dat" diff --git a/release-image/bootstrap.sh b/release-image/bootstrap.sh index 42b80764e383..ad415d70c477 100755 --- a/release-image/bootstrap.sh +++ b/release-image/bootstrap.sh @@ -7,7 +7,7 @@ function prepare_crs { echo_header "prepare crs for prover-agent image" local crs_src=${CRS_PATH:-$HOME/.bb-crs} - if [ ! -f "$crs_src/bn254_g1_compressed.dat" ]; then + if [ ! -f "$crs_src/bn254_g1.dat" ]; then # this assumes we pull the required number of points for proving the biggest circuit echo "CRS not found at $crs_src, downloading..." $root/barretenberg/scripts/download_bb_crs.sh @@ -15,7 +15,7 @@ function prepare_crs { fi mkdir -p crs - cp "$crs_src/bn254_g1_compressed.dat" crs/ + cp "$crs_src/bn254_g1.dat" crs/ cp "$crs_src/bn254_g2.dat" crs/ cp "$crs_src/grumpkin_g1.flat.dat" crs/ # Normalize timestamps so COPY --link produces an identical layer across builds