diff --git a/barretenberg/cpp/src/barretenberg/bbapi/bbapi_srs.cpp b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_srs.cpp index 0c886449656d..365eee1bb3ae 100644 --- a/barretenberg/cpp/src/barretenberg/bbapi/bbapi_srs.cpp +++ b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_srs.cpp @@ -15,14 +15,40 @@ namespace bb::bbapi { SrsInitSrs::Response SrsInitSrs::execute(BB_UNUSED BBApiRequest& request) && { - // Decompress 32-byte compressed points in parallel using native field arithmetic + constexpr size_t COMPRESSED_POINT_SIZE = 32; + constexpr size_t UNCOMPRESSED_POINT_SIZE = sizeof(g1::affine_element); // 64 + + size_t bytes_per_point = num_points > 0 ? points_buf.size() / num_points : 0; std::vector g1_points(num_points); - parallel_for([&](ThreadChunk chunk) { - for (auto i : chunk.range(static_cast(num_points))) { - uint256_t c = from_buffer(points_buf.data(), i * 32); - g1_points[i] = g1::affine_element::from_compressed(c); - } - }); + std::vector uncompressed_out; + + if (bytes_per_point == UNCOMPRESSED_POINT_SIZE) { + // Already uncompressed: fast path with from_buffer + parallel_for([&](ThreadChunk chunk) { + for (auto i : chunk.range(static_cast(num_points))) { + g1_points[i] = from_buffer(points_buf.data(), i * UNCOMPRESSED_POINT_SIZE); + } + }); + } else if (bytes_per_point == COMPRESSED_POINT_SIZE) { + // Compressed: decompress and return uncompressed bytes for caller to cache + parallel_for([&](ThreadChunk chunk) { + for (auto i : chunk.range(static_cast(num_points))) { + uint256_t c = from_buffer(points_buf.data(), i * COMPRESSED_POINT_SIZE); + g1_points[i] = g1::affine_element::from_compressed(c); + } + }); + // Serialize uncompressed points to return to caller for caching + uncompressed_out.resize(static_cast(num_points) * UNCOMPRESSED_POINT_SIZE); + parallel_for([&](ThreadChunk chunk) { + for (auto i : chunk.range(static_cast(num_points))) { + auto buf = to_buffer(g1_points[i]); + std::copy(buf.begin(), buf.end(), &uncompressed_out[i * UNCOMPRESSED_POINT_SIZE]); + } + }); + } else { + throw_or_abort("SrsInitSrs: invalid points_buf size. Expected 32 or 64 bytes per point, got " + + std::to_string(bytes_per_point)); + } // Parse G2 point from buffer (128 bytes) auto g2_point_elem = from_buffer(g2_point.data()); @@ -30,7 +56,7 @@ SrsInitSrs::Response SrsInitSrs::execute(BB_UNUSED BBApiRequest& request) && // Initialize BN254 SRS bb::srs::init_bn254_mem_crs_factory(g1_points, g2_point_elem); - return {}; + return { .points_buf = std::move(uncompressed_out) }; } SrsInitGrumpkinSrs::Response SrsInitGrumpkinSrs::execute(BB_UNUSED BBApiRequest& request) && diff --git a/barretenberg/cpp/src/barretenberg/bbapi/bbapi_srs.hpp b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_srs.hpp index 8d789b6d568e..f59fc3ab4357 100644 --- a/barretenberg/cpp/src/barretenberg/bbapi/bbapi_srs.hpp +++ b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_srs.hpp @@ -22,12 +22,13 @@ struct SrsInitSrs { struct Response { static constexpr const char MSGPACK_SCHEMA_NAME[] = "SrsInitSrsResponse"; - uint8_t dummy = 0; // Empty response needs a dummy field for msgpack - SERIALIZATION_FIELDS(dummy); + std::vector + points_buf; // Uncompressed G1 points (64 bytes each), empty if input was already uncompressed + SERIALIZATION_FIELDS(points_buf); bool operator==(const Response&) const = default; }; - std::vector points_buf; // G1 points (32 bytes each, compressed) + std::vector points_buf; // G1 points: compressed (32 bytes each) or uncompressed (64 bytes each) uint32_t num_points; std::vector g2_point; // G2 point (128 bytes) Response execute(BBApiRequest& request) &&; diff --git a/barretenberg/cpp/src/barretenberg/srs/factories/get_bn254_crs.cpp b/barretenberg/cpp/src/barretenberg/srs/factories/get_bn254_crs.cpp index db0c179ab55f..4b95fcefb2a9 100644 --- a/barretenberg/cpp/src/barretenberg/srs/factories/get_bn254_crs.cpp +++ b/barretenberg/cpp/src/barretenberg/srs/factories/get_bn254_crs.cpp @@ -18,6 +18,37 @@ constexpr const char* CRS_PRIMARY_URL = "http://crs.aztec-cdn.foundation/g1_comp // Fallback CRS URL (AWS S3) constexpr const char* CRS_FALLBACK_URL = "http://crs.aztec-labs.com/g1_compressed.dat"; constexpr size_t COMPRESSED_POINT_SIZE = 32; +constexpr size_t UNCOMPRESSED_POINT_SIZE = 64; // sizeof(g1::affine_element) + +/** + * @brief Write decompressed G1 points to a file in uncompressed format (64 bytes each). + */ +void write_uncompressed_g1_points(const std::vector& points, const std::filesystem::path& path) +{ + std::vector buf(points.size() * UNCOMPRESSED_POINT_SIZE); + bb::parallel_for([&](bb::ThreadChunk chunk) { + for (auto i : chunk.range(points.size())) { + auto serialized = to_buffer(points[i]); + std::copy(serialized.begin(), serialized.end(), &buf[i * UNCOMPRESSED_POINT_SIZE]); + } + }); + bb::write_file(path, buf); +} + +/** + * @brief Read uncompressed G1 points (64 bytes each) from a file. + */ +std::vector read_uncompressed_g1_points(const std::filesystem::path& path, size_t num_points) +{ + auto data = bb::read_file(path, num_points * UNCOMPRESSED_POINT_SIZE); + std::vector points(num_points); + bb::parallel_for([&](bb::ThreadChunk chunk) { + for (auto i : chunk.range(num_points)) { + points[i] = from_buffer(data, i * UNCOMPRESSED_POINT_SIZE); + } + }); + return points; +} /** * @brief Round num_points up to the next chunk boundary so every downloaded byte is hash-verified. @@ -166,20 +197,32 @@ std::vector get_bn254_g1_data(const std::filesystem::path& p BB_BENCH_NAME("get_bn254_g1_data"); std::filesystem::create_directories(path); + auto uncompressed_path = path / "bn254_g1.dat"; auto compressed_path = path / "bn254_g1_compressed.dat"; auto lock_path = path / "crs.lock"; // Acquire exclusive lock to prevent simultaneous downloads FileLockGuard lock(lock_path.string()); + // 1. Prefer cached uncompressed (fastest: parallel from_buffer, ~0.3s for 2^20 points) + size_t uncompressed_points = get_file_size(uncompressed_path) / UNCOMPRESSED_POINT_SIZE; + if (uncompressed_points >= num_points) { + vinfo("using cached uncompressed bn254 crs with ", uncompressed_points, " points at ", uncompressed_path); + return read_uncompressed_g1_points(uncompressed_path, num_points); + } + + // 2. Fall back to compressed on disk: decompress and cache uncompressed size_t compressed_points = get_file_size(compressed_path) / COMPRESSED_POINT_SIZE; if (compressed_points >= num_points) { - vinfo("using cached bn254 crs with ", std::to_string(compressed_points), " points at ", compressed_path); + vinfo("decompressing cached compressed bn254 crs (", compressed_points, " points)..."); auto data = read_file(compressed_path, num_points * COMPRESSED_POINT_SIZE); - return decompress_g1_points(data, num_points); + auto points = decompress_g1_points(data, num_points); + write_uncompressed_g1_points(points, uncompressed_path); + vinfo("cached uncompressed bn254 crs at ", uncompressed_path); + return points; } if (!allow_download && compressed_points == 0) { - throw_or_abort("bn254 g1 compressed data not found at " + compressed_path.string() + + throw_or_abort("bn254 g1 data not found at " + path.string() + " and bb does not automatically download in this context." + " Run barretenberg/crs/bootstrap.sh to download."); } else if (!allow_download) { @@ -191,16 +234,26 @@ std::vector get_bn254_g1_data(const std::filesystem::path& p } // Double-check after acquiring lock (another process may have downloaded while we waited) + uncompressed_points = get_file_size(uncompressed_path) / UNCOMPRESSED_POINT_SIZE; + if (uncompressed_points >= num_points) { + return read_uncompressed_g1_points(uncompressed_path, num_points); + } compressed_points = get_file_size(compressed_path) / COMPRESSED_POINT_SIZE; if (compressed_points >= num_points) { auto data = read_file(compressed_path, num_points * COMPRESSED_POINT_SIZE); - return decompress_g1_points(data, num_points); + auto points = decompress_g1_points(data, num_points); + write_uncompressed_g1_points(points, uncompressed_path); + return points; } + // 3. Download compressed, decompress, cache uncompressed vinfo("downloading bn254 crs..."); auto data = download_bn254_g1_data(num_points, primary_url, fallback_url); write_file(compressed_path, data); - return decompress_g1_points(data, num_points); + auto points = decompress_g1_points(data, num_points); + write_uncompressed_g1_points(points, uncompressed_path); + vinfo("cached uncompressed bn254 crs at ", uncompressed_path); + return points; } // Default overload using production URLs diff --git a/barretenberg/ts/src/barretenberg/index.ts b/barretenberg/ts/src/barretenberg/index.ts index 2290165aeb4a..f7021170415b 100644 --- a/barretenberg/ts/src/barretenberg/index.ts +++ b/barretenberg/ts/src/barretenberg/index.ts @@ -80,8 +80,16 @@ export class Barretenberg extends AsyncApi { const grumpkinCrs = await GrumpkinCrs.new(2 ** 16, this.options.crsPath, this.options.logger); // Load CRS into wasm global CRS state. - // TODO: Make RawBuffer be default behavior, and have a specific Vector type for when wanting length prefixed. - await this.srsInitSrs({ pointsBuf: crs.getG1Data(), numPoints: crs.numPoints, g2Point: crs.getG2Data() }); + // srsInitSrs auto-detects compressed (32B/point) vs uncompressed (64B/point). + // When decompressing, it returns the uncompressed bytes so we can cache them. + const response = await this.srsInitSrs({ + pointsBuf: crs.getG1Data(), + numPoints: crs.numPoints, + g2Point: crs.getG2Data(), + }); + if (response.pointsBuf.length > 0) { + await crs.cacheUncompressed(response.pointsBuf); + } await this.srsInitGrumpkinSrs({ pointsBuf: grumpkinCrs.getG1Data(), numPoints: grumpkinCrs.numPoints }); } diff --git a/barretenberg/ts/src/bbapi/exception_handling.test.ts b/barretenberg/ts/src/bbapi/exception_handling.test.ts index 3e95168623dc..11dd8eb7699d 100644 --- a/barretenberg/ts/src/bbapi/exception_handling.test.ts +++ b/barretenberg/ts/src/bbapi/exception_handling.test.ts @@ -47,7 +47,7 @@ describe('BBApi Exception Handling from bb.js', () => { expect(error).toBeInstanceOf(Error); expect((error as Error).message).toBeTruthy(); expect((error as Error).message.length).toBeGreaterThan(0); - expect((error as Error).message).toContain('g1_identity'); + expect((error as Error).message).toContain('invalid points_buf size'); console.log('Successfully caught exception from bb.js with message:', (error as Error).message); } }); diff --git a/barretenberg/ts/src/crs/browser/cached_net_crs.ts b/barretenberg/ts/src/crs/browser/cached_net_crs.ts index c50578643c1f..2c0acddda94d 100644 --- a/barretenberg/ts/src/crs/browser/cached_net_crs.ts +++ b/barretenberg/ts/src/crs/browser/cached_net_crs.ts @@ -19,19 +19,21 @@ export class CachedNetCrs { * Download the data. */ async init() { - const g1Compressed = await get('g1DataCompressed'); const g2Data = await get('g2Data'); - const netCrs = new NetCrs(this.numPoints); - const compressedLength = this.numPoints * 32; - if (g1Compressed && g1Compressed.length >= compressedLength) { - this.g1Data = g1Compressed; + // Prefer cached uncompressed (64 bytes/point, fast path: no decompression needed) + const g1Uncompressed = await get('g1Data'); + const uncompressedLength = this.numPoints * 64; + if (g1Uncompressed && g1Uncompressed.length >= uncompressedLength) { + this.g1Data = g1Uncompressed; } else { + // Download compressed from CDN + const netCrs = new NetCrs(this.numPoints); this.g1Data = await netCrs.downloadG1Data(); - await set('g1DataCompressed', this.g1Data); } if (!g2Data) { + const netCrs = new NetCrs(this.numPoints); this.g2Data = await netCrs.downloadG2Data(); await set('g2Data', this.g2Data); } else { @@ -40,12 +42,19 @@ export class CachedNetCrs { } /** - * G1 points data for prover key (compressed, 32 bytes/point). + * G1 points data for prover key (compressed or uncompressed). */ getG1Data(): Uint8Array { return this.g1Data; } + /** + * Cache uncompressed G1 data in IndexedDB after WASM decompression. + */ + async cacheUncompressed(data: Uint8Array): Promise { + await set('g1Data', data); + } + /** * G2 points data for verification key. * @returns The points data. diff --git a/barretenberg/ts/src/crs/node/index.ts b/barretenberg/ts/src/crs/node/index.ts index 75f911c54e2c..e8400661d441 100644 --- a/barretenberg/ts/src/crs/node/index.ts +++ b/barretenberg/ts/src/crs/node/index.ts @@ -26,23 +26,36 @@ export class Crs { return crs; } + private hasUncompressed = false; + async init(): Promise { mkdirSync(this.path, { recursive: true }); - const compressedFileSize = await stat(this.path + '/bn254_g1_compressed.dat') - .then(stats => stats.size) - .catch(() => 0); const g2FileSize = await stat(this.path + '/bn254_g2.dat') .then(stats => stats.size) .catch(() => 0); - const hasCompressed = compressedFileSize >= this.numPoints * 32 && compressedFileSize % 32 == 0; + // Prefer cached uncompressed (64 bytes/point, no decompression needed) + const uncompressedFileSize = await stat(this.path + '/bn254_g1.dat') + .then(stats => stats.size) + .catch(() => 0); + if (uncompressedFileSize >= this.numPoints * 64 && uncompressedFileSize % 64 == 0 && g2FileSize == 128) { + this.logger(`Using cached uncompressed CRS of size ${uncompressedFileSize / 64}`); + this.hasUncompressed = true; + return; + } - if (hasCompressed && g2FileSize == 128) { - this.logger(`Using cached compressed CRS of size ${compressedFileSize / 32}`); + // Fall back to compressed on disk + const compressedFileSize = await stat(this.path + '/bn254_g1_compressed.dat') + .then(stats => stats.size) + .catch(() => 0); + if (compressedFileSize >= this.numPoints * 32 && compressedFileSize % 32 == 0 && g2FileSize == 128) { + this.logger(`Using cached compressed CRS of size ${compressedFileSize / 32} (will decompress once)`); + this.hasUncompressed = false; return; } + // Download compressed from CDN this.logger(`Downloading CRS of size ${this.numPoints} into ${this.path}`); const crs = new NetCrs(this.numPoints); const g1Stream = await crs.streamG1Data(); @@ -52,14 +65,23 @@ export class Crs { finished(Readable.fromWeb(g1Stream as any).pipe(createWriteStream(this.path + '/bn254_g1_compressed.dat'))), finished(Readable.fromWeb(g2Stream as any).pipe(createWriteStream(this.path + '/bn254_g2.dat'))), ]); + this.hasUncompressed = false; } /** - * G1 points data for prover key (compressed, 32 bytes/point). - * Decompression happens in C++ via SrsInitSrs. + * G1 points data for prover key. Returns uncompressed (64 bytes/point) if cached, + * otherwise compressed (32 bytes/point) for WASM to decompress. */ getG1Data(): Uint8Array { const numPoints = Math.max(this.numPoints, 1); + if (this.hasUncompressed) { + const length = numPoints * 64; + const fd = openSync(this.path + '/bn254_g1.dat', 'r'); + const data = new Uint8Array(length); + readSync(fd, data, 0, length, 0); + closeSync(fd); + return data; + } const compressedLength = numPoints * 32; const fd = openSync(this.path + '/bn254_g1_compressed.dat', 'r'); const compressed = new Uint8Array(compressedLength); @@ -68,6 +90,14 @@ export class Crs { return compressed; } + /** + * Cache uncompressed G1 data to disk after WASM decompression. + */ + async cacheUncompressed(data: Uint8Array): Promise { + writeFileSync(this.path + '/bn254_g1.dat', data); + this.hasUncompressed = true; + } + /** * G2 points data for verification key. * @returns The points data.