From 74bac5989f0d8d2d4d0d5ed01407e91703132457 Mon Sep 17 00:00:00 2001 From: AztecBot Date: Thu, 3 Jul 2025 03:06:32 +0000 Subject: [PATCH 1/5] [empty] Start merge-train. Choo choo. From 18d5b23561c32b44c21ce8c8b3caf54715822c74 Mon Sep 17 00:00:00 2001 From: Jonathan Hao Date: Thu, 3 Jul 2025 15:09:43 +0200 Subject: [PATCH 2/5] feat: script to benchmark example flows remotely (#15469) I used this to get the benchmark results in https://github.com/AztecProtocol/aztec-packages/pull/15403#issuecomment-3028590921 --- .../benchmark_example_ivc_flow_remote.sh | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 barretenberg/cpp/scripts/benchmark_example_ivc_flow_remote.sh diff --git a/barretenberg/cpp/scripts/benchmark_example_ivc_flow_remote.sh b/barretenberg/cpp/scripts/benchmark_example_ivc_flow_remote.sh new file mode 100755 index 000000000000..07d6b4b6a4e3 --- /dev/null +++ b/barretenberg/cpp/scripts/benchmark_example_ivc_flow_remote.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -eu + +TARGET=${1:-"bb_cli_bench"} +FLOW=${2:-"deploy_ecdsar1+sponsored_fpc"} +BUILD_DIR="build-op-count-time" + +# Move above script dir. +cd $(dirname $0)/.. + +scp $BB_SSH_KEY ../../yarn-project/end-to-end/example-app-ivc-inputs-out/$FLOW/ivc-inputs.msgpack $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build/ + +# Measure the benchmarks with ops time counting +./scripts/benchmark_remote.sh "$TARGET"\ + "MAIN_ARGS='prove -o output --ivc_inputs_path ivc-inputs.msgpack --scheme client_ivc'\ + ./$TARGET --benchmark_out=$TARGET.json\ + --benchmark_out_format=json"\ + op-count-time\ + "$BUILD_DIR" + +# Retrieve output from benching instance +cd $BUILD_DIR +scp $BB_SSH_KEY $BB_SSH_INSTANCE:$BB_SSH_CPP_PATH/build/$TARGET.json . + +# Analyze the results +cd ../ +python3 ./scripts/analyze_client_ivc_bench.py --json "$TARGET.json" --benchmark "" --prefix "$BUILD_DIR" From 3cf54e58a63bcca2aef6e92d8f2d2da2f81f7862 Mon Sep 17 00:00:00 2001 From: Jonathan Hao Date: Thu, 3 Jul 2025 16:03:39 +0200 Subject: [PATCH 3/5] chore: Distribute content equally for each range (#15403) chrome: Distribute content equally among threads within each range The original code gives each thread the same number of rows but the same number of rows in each range does not necessarily mean the same work. This PR makes sure that for each range, each thread gets roughly the same number of rows. The leftovers are also evenly distributed because if we give them to the last thread for each range, the total worst case the last thread can get O(num_thread * num_ranges) more rows than others. --- .../execution_trace_usage_tracker.hpp | 56 +++++-------------- .../execution_trace_usage_tracker.test.cpp | 34 +++-------- 2 files changed, 24 insertions(+), 66 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/honk/execution_trace/execution_trace_usage_tracker.hpp b/barretenberg/cpp/src/barretenberg/honk/execution_trace/execution_trace_usage_tracker.hpp index fea9fb25e8a9..a0f6fde41d37 100644 --- a/barretenberg/cpp/src/barretenberg/honk/execution_trace/execution_trace_usage_tracker.hpp +++ b/barretenberg/cpp/src/barretenberg/honk/execution_trace/execution_trace_usage_tracker.hpp @@ -221,7 +221,10 @@ struct ExecutionTraceUsageTracker { * @brief Given a set of ranges indicating "active" regions of an ambient space, define a given number of new ranges * on the ambient space which evenly divide the content * @details In practive this is used to determine even distribution of execution trace rows across threads according - * to ranges describing the active rows of an IVC accumulator + * to ranges describing the active rows of an IVC accumulator. Even if two ranges contain the same number of rows, + * their workloads can differ depending on row complexity. To balance this, we distribute rows from each range as + * evenly as possible across the available threads. If the total number of rows is not perfectly divisible by the + * thread count, some threads will be assigned one additional row to ensure complete coverage. * * @param union_ranges A set of sorted, disjoint ranges * @param num_threads @@ -230,51 +233,22 @@ struct ExecutionTraceUsageTracker { static std::vector> construct_ranges_for_equal_content_distribution( const std::vector& union_ranges, const size_t num_threads) { - // Compute the minimum content per thread (final thread will get the leftovers = total_content % num_threads) - size_t total_content = 0; - for (const Range& range : union_ranges) { - total_content += range.second - range.first; - } - size_t content_per_thread = total_content / num_threads; - if (content_per_thread == 0) { // There are more threads than work - // Put all work at the back - std::vector> thread_ranges(num_threads); - thread_ranges.back() = union_ranges; - return thread_ranges; - } - - std::vector> thread_ranges = { {} }; - size_t thread_space_remaining = content_per_thread; // content space remaining in current thread + std::vector> thread_ranges(num_threads); for (const Range& range : union_ranges) { - size_t start_idx = range.first; - size_t content_to_distribute = range.second - start_idx; - while (content_to_distribute >= thread_space_remaining) { - // There's enough content to fill the remaining space in the current thread - size_t end_idx = start_idx + thread_space_remaining; - thread_ranges.back().push_back(Range{ start_idx, end_idx }); - start_idx = end_idx; - content_to_distribute -= thread_space_remaining; - thread_space_remaining = content_per_thread; - // Create a list for the next thread. - thread_ranges.push_back({}); + size_t total_content = range.second - start_idx; + size_t content_per_thread = total_content / num_threads; + size_t leftovers = total_content % num_threads; + for (size_t i = 0; i < num_threads; i++) { + size_t end_idx = + start_idx + content_per_thread + (i < leftovers ? 1 : 0); // Distribute leftovers evenly + if (start_idx < end_idx) { + thread_ranges[i].push_back(Range{ start_idx, end_idx }); + start_idx = end_idx; + } } - - if (content_to_distribute > 0) { - // Partially fill the next thread with all remaining content. - thread_ranges.back().push_back(Range{ start_idx, range.second }); - thread_space_remaining -= content_to_distribute; - } - } - - // Merge the last two sets of ranges because the last one is a left over. - std::vector back = std::move(thread_ranges.back()); - thread_ranges.pop_back(); - for (const Range& range : back) { - thread_ranges.back().push_back(range); } - thread_ranges.back() = construct_union_of_ranges(thread_ranges.back()); return thread_ranges; } diff --git a/barretenberg/cpp/src/barretenberg/honk/execution_trace/execution_trace_usage_tracker.test.cpp b/barretenberg/cpp/src/barretenberg/honk/execution_trace/execution_trace_usage_tracker.test.cpp index 6207e1898f4d..c177ba530b90 100644 --- a/barretenberg/cpp/src/barretenberg/honk/execution_trace/execution_trace_usage_tracker.test.cpp +++ b/barretenberg/cpp/src/barretenberg/honk/execution_trace/execution_trace_usage_tracker.test.cpp @@ -30,28 +30,12 @@ TEST_F(ExecutionTraceUsageTrackerTest, ConstructThreadRanges) { using Range = ExecutionTraceUsageTracker::Range; - std::vector union_ranges = { { 2, 8 }, { 13, 34 }, { 36, 42 }, { 50, 57 } }; + std::vector union_ranges = { { 2, 8 }, { 13, 34 }, { 36, 45 }, { 50, 60 } }; - std::vector> expected_thread_ranges = { - { { 2, 8 }, { 13, 17 } }, { { 17, 27 } }, { { 27, 34 }, { 36, 39 } }, { { 39, 42 }, { 50, 57 } } - }; - - const size_t num_threads = 4; - std::vector> thread_ranges = - ExecutionTraceUsageTracker::construct_ranges_for_equal_content_distribution(union_ranges, num_threads); - - EXPECT_EQ(thread_ranges, expected_thread_ranges); -} - -TEST_F(ExecutionTraceUsageTrackerTest, ConstructThreadRangesNotDivisible) -{ - using Range = ExecutionTraceUsageTracker::Range; - - std::vector union_ranges = { { 2, 8 }, { 13, 34 }, { 36, 42 }, { 50, 60 } }; - - std::vector> expected_thread_ranges = { - { { 2, 8 }, { 13, 17 } }, { { 17, 27 } }, { { 27, 34 }, { 36, 39 } }, { { 39, 42 }, { 50, 60 } } - }; + std::vector> expected_thread_ranges = { { { 2, 4 }, { 13, 19 }, { 36, 39 }, { 50, 53 } }, + { { 4, 6 }, { 19, 24 }, { 39, 41 }, { 53, 56 } }, + { { 6, 7 }, { 24, 29 }, { 41, 43 }, { 56, 58 } }, + { { 7, 8 }, { 29, 34 }, { 43, 45 }, { 58, 60 } } }; const size_t num_threads = 4; std::vector> thread_ranges = @@ -66,7 +50,7 @@ TEST_F(ExecutionTraceUsageTrackerTest, ConstructThreadRangesMoreThreadsThanWork) std::vector union_ranges = { { 2, 3 }, { 13, 14 } }; - std::vector> expected_thread_ranges = { {}, {}, {}, { { 2, 3 }, { 13, 14 } } }; + std::vector> expected_thread_ranges = { { { 2, 3 }, { 13, 14 } }, {}, {}, {} }; const size_t num_threads = 4; std::vector> thread_ranges = @@ -83,9 +67,9 @@ TEST_F(ExecutionTraceUsageTrackerTest, ConstructThreadRangesSplitsLargeRange) // Single range that is too large to fit in a single thread std::vector union_ranges = { { 0, 1 }, { 2, 101 } }; - std::vector> expected_thread_ranges = { { { 0, 1 }, { 2, 34 } }, - { { 34, 67 } }, - { { 67, 101 } } }; + std::vector> expected_thread_ranges = { { { 0, 1 }, { 2, 35 } }, + { { 35, 68 } }, + { { 68, 101 } } }; const size_t num_threads = 3; std::vector> thread_ranges = From a61ee4365cbc208466bf7d3da60ea6b13509bec2 Mon Sep 17 00:00:00 2001 From: Jonathan Hao Date: Thu, 3 Jul 2025 18:12:13 +0200 Subject: [PATCH 4/5] fix: update default benchmark in benchmark_remote.sh (#15500) `goblin_bench` does not seem to exist anymore. `client_ivc_bench` should be a sensible default. --- barretenberg/cpp/scripts/benchmark_remote.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/barretenberg/cpp/scripts/benchmark_remote.sh b/barretenberg/cpp/scripts/benchmark_remote.sh index b892d505ffba..c2240f07f4fa 100755 --- a/barretenberg/cpp/scripts/benchmark_remote.sh +++ b/barretenberg/cpp/scripts/benchmark_remote.sh @@ -7,7 +7,7 @@ # - BB_SSH_CPP_PATH: Path to barretenberg/cpp in a cloned repository on the EC2 instance set -eu -BENCHMARK=${1:-goblin_bench} +BENCHMARK=${1:-client_ivc_bench} COMMAND=${2:-./$BENCHMARK} PRESET=${3:-clang16} BUILD_DIR=${4:-build} From b14dbc4dfea5bca1cbfbd5dd1e3e32429205ac5d Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 3 Jul 2025 17:22:58 -0400 Subject: [PATCH 5/5] feat(bb): add subset of bbrpc. test api_client_ivc.cpp (#15476) Picks out a ready-to-merge chunk of bbrpc functionality. bbrpc_commands.hpp defining RPC command structures named_union.hpp for command variant handling bbrpc_execute.hpp doing some basic execution added the first acir-serializing tests to api_client_ivc.test.cpp --------- Co-authored-by: AztecBot Co-authored-by: ledwards2225 <98505400+ledwards2225@users.noreply.github.com> --- .vscode/launch.json | 28 +- .../src/barretenberg/api/api_client_ivc.cpp | 3 +- .../barretenberg/api/api_client_ivc.test.cpp | 305 +++++++++++ .../src/barretenberg/api/bbrpc_commands.hpp | 515 ++++++++++++++++++ .../src/barretenberg/api/bbrpc_execute.hpp | 219 ++++++++ .../client_ivc/acir_bincode_mocks.hpp | 126 +++++ .../client_ivc/mock_circuit_producer.hpp | 2 +- .../client_ivc/private_execution_steps.cpp | 62 ++- .../client_ivc/private_execution_steps.hpp | 4 + .../src/barretenberg/common/named_union.hpp | 155 ++++++ .../stdlib_circuit_builders/mock_circuits.hpp | 6 +- 11 files changed, 1416 insertions(+), 9 deletions(-) create mode 100644 barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp create mode 100644 barretenberg/cpp/src/barretenberg/api/bbrpc_commands.hpp create mode 100644 barretenberg/cpp/src/barretenberg/api/bbrpc_execute.hpp create mode 100644 barretenberg/cpp/src/barretenberg/client_ivc/acir_bincode_mocks.hpp create mode 100644 barretenberg/cpp/src/barretenberg/common/named_union.hpp diff --git a/.vscode/launch.json b/.vscode/launch.json index cc3610fac124..d19847edad1b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -44,11 +44,33 @@ ], }, { - "name": "Debug Test", + "name": "Debug BB API Test", "type": "lldb", "request": "launch", - "program": "${workspaceFolder}/barretenberg/cpp/build-debug/bin/vm2_tests", - "args": ["--gtest_filter=AvmRecursiveTests.GoblinRecursion"], + "program": "${workspaceFolder}/barretenberg/cpp/build-debug/bin/api_tests", + "args": ["--gtest_filter=ClientIVCAPITests.ProveAndVerifyFileBasedFlow"], + "cwd": "${workspaceFolder}/barretenberg/cpp/build-debug", + "initCommands": [ + "command script import ${workspaceFolder}/barretenberg/cpp/scripts/lldb_format.py" + ], + }, + { + "name": "Debug BB ClientIVC Test", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/barretenberg/cpp/build-debug/bin/client_ivc_tests", + "args": ["--gtest_filter=ClientIVCTests.ProveAndVerifyFileBasedFlow"], + "cwd": "${workspaceFolder}/barretenberg/cpp/build-debug", + "initCommands": [ + "command script import ${workspaceFolder}/barretenberg/cpp/scripts/lldb_format.py" + ], + }, + { + "name": "Debug BB DSL Test", + "type": "lldb", + "request": "launch", + "program": "${workspaceFolder}/barretenberg/cpp/build-debug/bin/dsl_tests", + "args": ["--gtest_filter=IvcRecursionConstraintTest.AccumulateTwo"], "cwd": "${workspaceFolder}/barretenberg/cpp/build-debug", "initCommands": [ "command script import ${workspaceFolder}/barretenberg/cpp/scripts/lldb_format.py" diff --git a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp index a20a9b3dab09..7c063a0c50cc 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp +++ b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp @@ -181,8 +181,7 @@ bool ClientIVCAPI::prove_and_verify(const std::filesystem::path& input_path) return verified; } -void ClientIVCAPI::gates([[maybe_unused]] const Flags& flags, - [[maybe_unused]] const std::filesystem::path& bytecode_path) +void ClientIVCAPI::gates(const Flags& flags, const std::filesystem::path& bytecode_path) { gate_count_for_ivc(bytecode_path, flags.include_gates_per_opcode); } diff --git a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp new file mode 100644 index 000000000000..6b2f4d97d6ae --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp @@ -0,0 +1,305 @@ +#include "api_client_ivc.hpp" +#include "barretenberg/api/bbrpc_commands.hpp" +#include "barretenberg/api/bbrpc_execute.hpp" +#include "barretenberg/api/file_io.hpp" +#include "barretenberg/client_ivc/acir_bincode_mocks.hpp" +#include "barretenberg/client_ivc/client_ivc.hpp" +#include "barretenberg/client_ivc/mock_circuit_producer.hpp" +#include "barretenberg/client_ivc/private_execution_steps.hpp" +#include "barretenberg/common/serialize.hpp" +#include "barretenberg/dsl/acir_format/acir_format.hpp" +#include "barretenberg/dsl/acir_format/acir_format_mocks.hpp" +#include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp" +#include "barretenberg/dsl/acir_format/serde/acir.hpp" +#include "barretenberg/dsl/acir_format/serde/witness_stack.hpp" +#include "barretenberg/flavor/mega_flavor.hpp" +#include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/srs/global_crs.hpp" +#include +#include +#include +#include +#include +#include +#include + +using namespace bb; + +namespace { +// Create a unique temporary directory for each test run +// Uniqueness needed because tests are run in parallel and write to same file names. +std::filesystem::path get_test_dir(const std::string_view& test_name) +{ + std::filesystem::path temp_dir = "tmp_api_client_ivc_test"; + std::filesystem::create_directories(temp_dir); + std::filesystem::create_directories(temp_dir / test_name); + return temp_dir / test_name; +} + +void create_test_private_execution_steps(const std::filesystem::path& output_path) +{ + using namespace acir_format; + + // First create a simple app circuit + auto [app_bytecode, app_witness_data] = acir_bincode_mocks::create_simple_circuit_bytecode(); + + // Get the VK for the app circuit + bbrpc::BBRpcRequest request; + + auto app_vk = + bbrpc::execute(request, + bbrpc::ClientIvcComputeVk{ .circuit = { .name = "app_circuit", .bytecode = app_bytecode }, + .standalone = true }) + .verification_key; + auto app_vk_fields = from_buffer(app_vk).to_field_elements(); + + // Now create a kernel circuit that verifies the app circuit + auto kernel_bytecode = acir_bincode_mocks::create_simple_kernel(app_vk_fields.size(), /*is_init_kernel=*/true); + auto kernel_witness_data = acir_bincode_mocks::create_kernel_witness(app_vk_fields); + + auto kernel_vk = + bbrpc::execute(request, + bbrpc::ClientIvcComputeVk{ .circuit = { .name = "kernel_circuit", .bytecode = kernel_bytecode }, + .standalone = true }) + .verification_key; + + // Create PrivateExecutionStepRaw for the kernel + std::vector raw_steps; + raw_steps.push_back( + { .bytecode = app_bytecode, .witness = app_witness_data, .vk = app_vk, .function_name = "app_function" }); + raw_steps.push_back({ .bytecode = kernel_bytecode, + .witness = kernel_witness_data, + .vk = kernel_vk, + .function_name = "kernel_function" }); + PrivateExecutionStepRaw::compress_and_save(std::move(raw_steps), output_path); +} +} // namespace + +class ClientIVCAPITests : public ::testing::Test { + protected: + static void SetUpTestSuite() { bb::srs::init_file_crs_factory(bb::srs::bb_crs_path()); } + + void SetUp() override + { + const auto* info = ::testing::UnitTest::GetInstance()->current_test_info(); + test_dir = get_test_dir(info->name()); + } + + void TearDown() override + { + if (std::filesystem::exists(test_dir)) { + std::filesystem::remove_all(test_dir); + } + } + + std::filesystem::path test_dir; +}; + +namespace bb { +std::vector compress(const std::vector& input); +} + +// Used to get a mock IVC vk. +ClientIVC::MegaVerificationKey get_ivc_vk(const std::filesystem::path& test_dir) +{ + auto [app_bytecode, app_witness_data] = acir_bincode_mocks::create_simple_circuit_bytecode(); + bbrpc::BBRpcRequest request; + // First create an app standalone VK. + auto app_vk = + bbrpc::execute(request, + bbrpc::ClientIvcComputeVk{ .circuit = { .name = "app_circuit", .bytecode = app_bytecode }, + .standalone = true }) + .verification_key; + auto app_vk_fields = from_buffer(app_vk).to_field_elements(); + // Use this to get the size of the vk. + auto bytecode = acir_bincode_mocks::create_simple_kernel(app_vk_fields.size(), /*is_init_kernel=*/false); + std::filesystem::path bytecode_path = test_dir / "circuit.acir"; + write_file(bytecode_path, bb::compress(bytecode)); + + ClientIVCAPI::Flags write_vk_flags; + write_vk_flags.verifier_type = "ivc"; + write_vk_flags.output_format = "bytes"; + + ClientIVCAPI api; + api.write_vk(write_vk_flags, bytecode_path, test_dir); + + return from_buffer(read_file(test_dir / "vk")); +}; + +// Test the ClientIVCAPI::prove flow, making sure --write_vk +// returns the same output as our ivc VK generation. +TEST_F(ClientIVCAPITests, ProveAndVerifyFileBasedFlow) +{ + auto ivc_vk = get_ivc_vk(test_dir); + + // Create test input file + std::filesystem::path input_path = test_dir / "input.msgpack"; + create_test_private_execution_steps(input_path); + + std::filesystem::path output_dir = test_dir / "output"; + std::filesystem::create_directories(output_dir); + + // Helper lambda to create proof and VK files + auto create_proof_and_vk = [&]() { + ClientIVCAPI::Flags flags; + flags.write_vk = true; + + ClientIVCAPI api; + api.prove(flags, input_path, output_dir); + }; + + // Helper lambda to verify VK equivalence + auto verify_vk_equivalence = [&](const std::filesystem::path& vk1_path, const ClientIVC::MegaVerificationKey& vk2) { + auto vk1_data = read_file(vk1_path); + auto vk1 = from_buffer(vk1_data); + ASSERT(msgpack::msgpack_check_eq(vk1, vk2, "VK from prove should match VK from write_vk")); + }; + + // Helper lambda to verify proof + auto verify_proof = [&]() { + std::filesystem::path proof_path = output_dir / "proof"; + std::filesystem::path vk_path = output_dir / "vk"; + std::filesystem::path public_inputs_path; // Not used for ClientIVC + + ClientIVCAPI::Flags flags; + ClientIVCAPI verify_api; + return verify_api.verify(flags, public_inputs_path, proof_path, vk_path); + }; + + // Execute test steps + create_proof_and_vk(); + verify_vk_equivalence(output_dir / "vk", ivc_vk); + // Test verify command + EXPECT_TRUE(verify_proof()); +} + +// WORKTODO(bbrpc): Expand on this. +TEST_F(ClientIVCAPITests, WriteVkFieldsSmokeTest) +{ + // Create a simple circuit bytecode + auto [bytecode, witness_data] = acir_bincode_mocks::create_simple_circuit_bytecode(); + + // Compress and write bytecode to file + std::filesystem::path bytecode_path = test_dir / "circuit.acir"; + write_file(bytecode_path, bb::compress(bytecode)); + + // Test write_vk with fields output format + ClientIVCAPI::Flags flags; + flags.verifier_type = "standalone"; + flags.output_format = "fields"; + + ClientIVCAPI api; + api.write_vk(flags, bytecode_path, test_dir); + + // Read and verify the fields format + auto vk_data = read_file(test_dir / "vk_fields.json"); + std::string vk_str(vk_data.begin(), vk_data.end()); + // Just check that this looks a bit like JSON. + EXPECT_NE(vk_str.find('['), std::string::npos); + EXPECT_NE(vk_str.find(']'), std::string::npos); +} + +// TODO(https://github.com/AztecProtocol/barretenberg/issues/1461): Make this test actually test # gates +TEST_F(ClientIVCAPITests, GatesCommandSmokeTest) +{ + // Create a simple circuit bytecode + auto [bytecode, witness_data] = acir_bincode_mocks::create_simple_circuit_bytecode(); + + // Write compressed bytecode to file + std::filesystem::path bytecode_path = test_dir / "circuit.acir"; + write_file(bytecode_path, bb::compress(bytecode)); + + ClientIVCAPI::Flags flags; + flags.include_gates_per_opcode = true; + + // Redirect stdout to a stringstream + std::ostringstream captured_output; + std::streambuf* old_cout = std::cout.rdbuf(captured_output.rdbuf()); + + ClientIVCAPI api; + api.gates(flags, bytecode_path); + + // Restore stdout + std::cout.rdbuf(old_cout); + std::string output = captured_output.str(); + + // We rudimentarily output to this pattern: + // {"functions": [ + // { + // "acir_opcodes": 1, + // "circuit_size": *, + // "gates_per_opcode": [*] + // } + // ]} + EXPECT_NE(output.find("\"functions\": ["), std::string::npos); + EXPECT_NE(output.find("\"acir_opcodes\": 1"), std::string::npos); + EXPECT_NE(output.find("\"circuit_size\": "), std::string::npos); + EXPECT_NE(output.find("\"gates_per_opcode\": ["), std::string::npos); +} + +// Test prove_and_verify for our example IVC flow. +TEST_F(ClientIVCAPITests, ProveAndVerifyCommand) +{ + // Create test input file + std::filesystem::path input_path = test_dir / "input.msgpack"; + create_test_private_execution_steps(input_path); + + ClientIVCAPI api; + EXPECT_TRUE(api.prove_and_verify(input_path)); +} + +// Check a case where precomputed VKs match +TEST_F(ClientIVCAPITests, CheckPrecomputedVks) +{ + // Create test input file with precomputed VKs + std::filesystem::path input_path = test_dir / "input_with_vks.msgpack"; + create_test_private_execution_steps(input_path); + + ClientIVCAPI api; + EXPECT_TRUE(api.check_precomputed_vks(input_path)); +} + +// Check a case where precomputed VKs don't match +TEST_F(ClientIVCAPITests, CheckPrecomputedVksMismatch) +{ + using namespace acir_format; + + // Create a simple circuit + auto [bytecode, witness_data] = acir_bincode_mocks::create_simple_circuit_bytecode(); + + bbrpc::BBRpcRequest request; + size_t vk_size = + from_buffer( + bbrpc::execute(request, + bbrpc::ClientIvcComputeVk{ .circuit = { .name = "simple_circuit", .bytecode = bytecode }, + .standalone = true }) + .verification_key) + .to_field_elements() + .size(); + + // Create a WRONG verification key (use a different circuit) + auto different_bytecode = acir_bincode_mocks::create_simple_kernel(vk_size, /*is_init_kernel=*/true); + auto vk = bbrpc::execute( + request, + bbrpc::ClientIvcComputeVk{ .circuit = { .name = "different_circuit", .bytecode = different_bytecode }, + .standalone = true }) + .verification_key; + + // Create PrivateExecutionStepRaw with wrong VK + std::vector raw_steps; + PrivateExecutionStepRaw step; + step.bytecode = bytecode; + step.witness = witness_data; + step.vk = std::move(vk); // Wrong VK + step.function_name = "test_function"; + raw_steps.push_back(std::move(step)); + + // Write to file using compress_and_save + std::filesystem::path input_path = test_dir / "input_wrong_vks.msgpack"; + PrivateExecutionStepRaw::compress_and_save(std::move(raw_steps), input_path); + + // Should fail because VK doesn't match + ClientIVCAPI api; + bool result = api.check_precomputed_vks(input_path); + EXPECT_FALSE(result); +} diff --git a/barretenberg/cpp/src/barretenberg/api/bbrpc_commands.hpp b/barretenberg/cpp/src/barretenberg/api/bbrpc_commands.hpp new file mode 100644 index 000000000000..1963a7961050 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/api/bbrpc_commands.hpp @@ -0,0 +1,515 @@ +#pragma once +/** + * @file bbrpc.hpp + * @brief Barretenberg RPC provides a stateful API for all core barretenberg proving functions. + * Not included: + * - Solidity verifier generation + * - Raw cryptography functions exposed by WASM BB + */ +#include "barretenberg/client_ivc/client_ivc.hpp" +#include "barretenberg/common/named_union.hpp" +#include "barretenberg/honk/proof_system/types/proof.hpp" +#include + +namespace bb::bbrpc { + +/** + * @struct CircuitInputNoVK + * @brief A circuit to be used in either ultrahonk or chonk (ClientIVC+honk) verification key derivation. + */ +struct CircuitInputNoVK { + /** + * @brief Human-readable name for the circuit + * + * This name is not used for processing but serves as a debugging aid and + * provides context for circuit identification in logs and diagnostics. + */ + std::string name; + + /** + * @brief Serialized bytecode representation of the circuit + * + * Contains the ACIR program in serialized form. The format (bincode or msgpack) + * is determined by examining the first byte of the bytecode. + */ + std::vector bytecode; +}; + +/** + * @struct CircuitInput + * @brief A circuit to be used in either ultrahonk or ClientIVC-honk proving. + */ +struct CircuitInput { + /** + * @brief Human-readable name for the circuit + * + * This name is not used for processing but serves as a debugging aid and + * provides context for circuit identification in logs and diagnostics. + */ + std::string name; + + /** + * @brief Serialized bytecode representation of the circuit + * + * Contains the ACIR program in serialized form. The format (bincode or msgpack) + * is determined by examining the first byte of the bytecode. + */ + std::vector bytecode; + + /** + * @brief Verification key of the circuit. This could be derived, but it is more efficient to have it fixed ahead of + * time. As well, this guards against unexpected changes in the verification key. + */ + std::vector verification_key; +}; + +struct ProofSystemSettings { + /** + * @brief Optional flag to indicate if the proof should be generated with IPA accumulation (i.e. for rollup + * circuits). + */ + bool ipa_accumulation = false; + + /** + * @brief The oracle hash type to be used for the proof. + * + * This is used to determine the hash function used in the proof generation. + * Valid values are "poseidon2", "keccak", and "starknet". + */ + std::string oracle_hash_type = "poseidon2"; + + /** + * @brief Flag to disable blinding of the proof. + * Useful for cases that don't require privacy, such as when all inputs are public or zk-SNARK proofs themselves. + */ + bool disable_zk = false; + + /** + * @brief Honk recursion setting. + * 0 = no recursion, 1 = UltraHonk recursion, 2 = UltraRollupHonk recursion. + * Controls whether pairing point accumulators and IPA claims are added to public inputs. + */ + uint32_t honk_recursion = 0; + + /** + * @brief Flag to indicate if this circuit will be recursively verified. + */ + bool recursive = false; +}; + +/** + * @struct CircuitProve + * @brief Represents a request to generate a proof. + * Currently, UltraHonk is the only proving system supported by BB (after plonk was deprecated and removed). + * This is used for one-shot proving, not our "IVC" scheme, ClientIVC-honk. For that, use the ClientIVC* commands. + * + * This structure is used to encapsulate all necessary parameters for generating a proof + * for a specific circuit, including the circuit bytecode, verification key, witness data, and options for the proving + * process. + */ +struct CircuitProve { + static constexpr const char* NAME = "CircuitProve"; + + /** + * @brief Contains proof and public inputs. + * Both are given as vectors of fields. To be used for verification. + * Example uses of this Response would be verification in native BB, WASM BB, solidity or recursively through Noir. + */ + struct Response { + static constexpr const char* NAME = "CircuitProveResponse"; + + PublicInputsVector public_inputs; + HonkProof proof; + // Empty if successful. + std::string error_message; + }; + + CircuitInput circuit; + std::vector witness; + ProofSystemSettings settings; +}; + +struct CircuitComputeVk { + static constexpr const char* NAME = "CircuitComputeVk"; + + struct Response { + static constexpr const char* NAME = "CircuitComputeVkResponse"; + + /** + * @brief Serialized verification key. + */ + std::vector verification_key; + // Empty if successful. + std::string error_message; + }; + + CircuitInputNoVK circuit; + ProofSystemSettings settings; +}; + +/** Compute verification key, Treat the previously loaded circuit as either a standalone circuit + * or a common final circuit used to verify all of IVC. */ +struct CircuitComputeIvcVk { + static constexpr const char* NAME = "CircuitComputeIvcVk"; + + struct Response { + static constexpr const char* NAME = "CircuitComputeIvcVkResponse"; + + /** + * @brief Serialized verification key. + */ + std::vector verification_key; + // Empty if successful. + std::string error_message; + }; + bool standalone; +}; + +/** Compute verification key, Treat the previously loaded circuit as either a standalone circuit + * or a common final circuit used to verify all of IVC. */ +struct ClientIvcComputeVk { + static constexpr const char* NAME = "ClientIvcComputeVk"; + + struct Response { + static constexpr const char* NAME = "ClientIvcComputeVkResponse"; + + /** + * @brief Serialized verification key. + */ + std::vector verification_key; + // Empty if successful. + std::string error_message; + }; + + CircuitInputNoVK circuit; + bool standalone; +}; + +/** + * @brief + * Note, only one IVC request can be made at a time for each batch_request. + */ +struct ClientIvcStart { + static constexpr const char* NAME = "ClientIvcStart"; + + struct Response { + static constexpr const char* NAME = "ClientIvcStartResponse"; + + // Empty if successful. + std::string error_message; + }; +}; + +struct ClientIvcLoad { + static constexpr const char* NAME = "ClientIvcLoad"; + + struct Response { + static constexpr const char* NAME = "ClientIvcLoadResponse"; + + // Empty if successful. + std::string error_message; + }; + + CircuitInput circuit; +}; + +struct ClientIvcAccumulate { + static constexpr const char* NAME = "ClientIvcAccumulate"; + + struct Response { + static constexpr const char* NAME = "ClientIvcAccumulateResponse"; + + // Empty if successful. + std::string error_message; + }; + + // Serialized witness for the last loaded circuit. + std::vector witness; +}; + +struct ClientIvcProve { + static constexpr const char* NAME = "ClientIvcProve"; + + struct Response { + static constexpr const char* NAME = "ClientIvcProveResponse"; + + ClientIVC::Proof proof; + // Empty if successful. + std::string error_message; + }; +}; + +/** + * @struct CircuitInfo + * @brief Consolidated command for retrieving circuit information. + * Combines gate count, circuit size, and other metadata into a single command. + */ +struct CircuitInfo { + static constexpr const char* NAME = "CircuitInfo"; + + struct Response { + static constexpr const char* NAME = "CircuitInfoResponse"; + + uint32_t total_gates; + uint32_t subgroup_size; + // Optional: gate counts per opcode + std::map gates_per_opcode; + // Empty if successful. + std::string error_message; + }; + + CircuitInput circuit; + bool include_gates_per_opcode = false; + ProofSystemSettings settings; +}; + +/** + * @struct CircuitCheck + * @brief Verify that a witness satisfies a circuit's constraints. + * For debugging and validation purposes. + */ +struct CircuitCheck { + static constexpr const char* NAME = "CircuitCheck"; + + struct Response { + static constexpr const char* NAME = "CircuitCheckResponse"; + + bool satisfied; + // Empty if successful, contains constraint failure details if not satisfied. + std::string error_message; + }; + + CircuitInput circuit; + std::vector witness; + ProofSystemSettings settings; +}; + +/** + * @struct CircuitVerify + * @brief Verify a proof against a verification key and public inputs. + */ +struct CircuitVerify { + static constexpr const char* NAME = "CircuitVerify"; + + struct Response { + static constexpr const char* NAME = "CircuitVerifyResponse"; + + bool verified; + // Empty if successful. + std::string error_message; + }; + + std::vector verification_key; + PublicInputsVector public_inputs; + HonkProof proof; + ProofSystemSettings settings; +}; + +/** + * @struct ProofAsFields + * @brief Convert a proof to field elements representation. + */ +struct ProofAsFields { + static constexpr const char* NAME = "ProofAsFields"; + + struct Response { + static constexpr const char* NAME = "ProofAsFieldsResponse"; + + std::vector fields; + // Empty if successful. + std::string error_message; + }; + + HonkProof proof; +}; + +/** + * @struct VkAsFields + * @brief Convert a verification key to field elements representation. + */ +struct VkAsFields { + static constexpr const char* NAME = "VkAsFields"; + + struct Response { + static constexpr const char* NAME = "VkAsFieldsResponse"; + + std::vector fields; + // Empty if successful. + std::string error_message; + }; + + std::vector verification_key; + bool is_mega_honk = false; +}; + +/** + * @brief Command to generate Solidity verifier contract + */ +struct CircuitWriteSolidityVerifier { + static constexpr const char* NAME = "CircuitWriteSolidityVerifier"; + + struct Response { + static constexpr const char* NAME = "CircuitWriteSolidityVerifierResponse"; + + std::string solidity_code; + std::string error_message; + }; + + std::vector verification_key; + ProofSystemSettings settings; +}; + +/** + * @brief Command to prove and verify in one step + */ +struct CircuitProveAndVerify { + static constexpr const char* NAME = "CircuitProveAndVerify"; + + struct Response { + static constexpr const char* NAME = "CircuitProveAndVerifyResponse"; + + bool verified; + std::vector proof; // The generated proof + std::vector public_inputs; // Extracted public inputs + std::string error_message; + }; + + CircuitInput circuit; + std::vector witness; + ProofSystemSettings settings; +}; + +/** + * @brief Command to write circuit bytecode in various formats + */ +struct CircuitWriteBytecode { + static constexpr const char* NAME = "CircuitWriteBytecode"; + + struct Response { + static constexpr const char* NAME = "CircuitWriteBytecodeResponse"; + + std::vector bytecode; + std::string formatted_output; // For hex/base64 + std::string error_message; + }; + + CircuitInput circuit; + std::string format = "binary"; // binary, hex, base64 +}; + +/** + * @brief Command to validate circuit structure + */ +struct CircuitValidate { + static constexpr const char* NAME = "CircuitValidate"; + + struct Response { + static constexpr const char* NAME = "CircuitValidateResponse"; + + bool is_valid; + std::vector validation_errors; + std::string error_message; + }; + + CircuitInput circuit; + ProofSystemSettings settings; + bool check_recursive_structure = false; +}; + +/** + * @brief Command to benchmark circuit operations + */ +struct CircuitBenchmark { + static constexpr const char* NAME = "CircuitBenchmark"; + + struct Response { + static constexpr const char* NAME = "CircuitBenchmarkResponse"; + + double witness_generation_time_ms; + double proving_time_ms; + double verification_time_ms; + uint64_t peak_memory_bytes; + std::string error_message; + }; + + CircuitInput circuit; + std::vector witness; + ProofSystemSettings settings; + uint32_t num_iterations = 1; + bool benchmark_witness_generation = true; + bool benchmark_proving = true; +}; + +/** + * @brief Command to check if a precomputed VK matches the circuit + */ +struct ClientIvcCheckPrecomputedVk { + static constexpr const char* NAME = "ClientIvcCheckPrecomputedVk"; + + struct Response { + static constexpr const char* NAME = "ClientIvcCheckPrecomputedVkResponse"; + + bool valid; + std::string error_message; + }; + + // Circuit with its precomputed VK + CircuitInput circuit; + std::string function_name; +}; + +using Command = NamedUnion; + +using CommandResponse = NamedUnion; + +/** + * @brief Convert oracle hash type string to enum for internal use + */ +enum class OracleHashType { POSEIDON2, KECCAK, STARKNET }; + +inline OracleHashType parse_oracle_hash_type(const std::string& type) +{ + if (type == "keccak") { + return OracleHashType::KECCAK; + } + if (type == "starknet") { + return OracleHashType::STARKNET; + } + return OracleHashType::POSEIDON2; // default +} + +} // namespace bb::bbrpc diff --git a/barretenberg/cpp/src/barretenberg/api/bbrpc_execute.hpp b/barretenberg/cpp/src/barretenberg/api/bbrpc_execute.hpp new file mode 100644 index 000000000000..d2f81823468d --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/api/bbrpc_execute.hpp @@ -0,0 +1,219 @@ +#pragma once + +#include "barretenberg/api/bbrpc_commands.hpp" +#include "barretenberg/api/write_prover_output.hpp" +#include "barretenberg/circuit_checker/circuit_checker.hpp" +#include "barretenberg/client_ivc/client_ivc.hpp" +#include "barretenberg/client_ivc/mock_circuit_producer.hpp" +#include "barretenberg/common/compiler_hints.hpp" +#include "barretenberg/common/log.hpp" +#include "barretenberg/common/throw_or_abort.hpp" +#include "barretenberg/dsl/acir_format/acir_format.hpp" +#include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp" +#include "barretenberg/dsl/acir_format/serde/witness_stack.hpp" +#include "barretenberg/honk/execution_trace/mega_execution_trace.hpp" +#include "barretenberg/serialize/msgpack_check_eq.hpp" +#include "barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp" +#include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp" +#include +#include + +namespace bb::bbrpc { + +struct BBRpcRequest { + TraceSettings trace_settings{ AZTEC_TRACE_STRUCTURE }; + // Current depth of the IVC stack for this request + uint32_t ivc_stack_depth = 0; + std::shared_ptr ivc_in_progress; + // Name of the last loaded circuit + std::string last_circuit_name; + // Store the parsed constraint system to get ahead of parsing before accumulate + std::optional last_circuit_constraints; + // Store the verification key passed with the circuit + std::vector last_circuit_vk; +}; + +inline const std::string& get_error_message(const CommandResponse& response) +{ + return response.visit([](const auto& resp) -> const std::string& { return resp.error_message; }); +} + +inline CircuitProve::Response execute(BB_UNUSED BBRpcRequest& request, CircuitProve&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +inline CircuitComputeVk::Response execute(BB_UNUSED BBRpcRequest& request, CircuitComputeVk&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +inline CircuitVerify::Response execute(BB_UNUSED BBRpcRequest& request, CircuitVerify&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +inline CircuitInfo::Response execute(BB_UNUSED BBRpcRequest& request, CircuitInfo&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +inline CircuitCheck::Response execute(BB_UNUSED BBRpcRequest& request, CircuitCheck&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +inline ProofAsFields::Response execute(BB_UNUSED BBRpcRequest& request, ProofAsFields&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +inline VkAsFields::Response execute(BB_UNUSED BBRpcRequest& request, VkAsFields&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +inline ClientIvcStart::Response execute(BBRpcRequest& request, BB_UNUSED ClientIvcStart&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +inline ClientIvcLoad::Response execute(BBRpcRequest& request, ClientIvcLoad&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +inline ClientIvcAccumulate::Response execute(BBRpcRequest& request, ClientIvcAccumulate&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +inline ClientIvcProve::Response execute(BBRpcRequest& request, ClientIvcProve&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +inline std::shared_ptr get_acir_program_decider_proving_key( + const BBRpcRequest& request, acir_format::AcirProgram& program) +{ + ClientIVC::ClientCircuit builder = acir_format::create_circuit(program); + + // Construct the verification key via the prover-constructed proving key with the proper trace settings + return std::make_shared(builder, request.trace_settings); +} + +inline ClientIVC::VerificationKey compute_vk_for_ivc(const BBRpcRequest& request, + size_t num_public_inputs_in_final_circuit) +{ + ClientIVC ivc{ request.trace_settings }; + ClientIVCMockCircuitProducer circuit_producer; + + // Initialize the IVC with an arbitrary circuit + // We segfault if we only call accumulate once + static constexpr size_t SMALL_ARBITRARY_LOG_CIRCUIT_SIZE{ 5 }; + MegaCircuitBuilder circuit_0 = circuit_producer.create_next_circuit(ivc, SMALL_ARBITRARY_LOG_CIRCUIT_SIZE); + ivc.accumulate(circuit_0); + + // Create another circuit and accumulate + MegaCircuitBuilder circuit_1 = + circuit_producer.create_next_circuit(ivc, SMALL_ARBITRARY_LOG_CIRCUIT_SIZE, num_public_inputs_in_final_circuit); + ivc.accumulate(circuit_1); + + // Construct the hiding circuit and its VK (stored internally in the IVC) + ivc.construct_hiding_circuit_key(); + + return ivc.get_vk(); +} + +inline ClientIvcComputeVk::Response execute(BBRpcRequest& request, ClientIvcComputeVk&& command) +{ + info("ClientIvcComputeVk - deriving VK for circuit '", command.circuit.name, "', standalone: ", command.standalone); + + // Parse the circuit + auto constraint_system = acir_format::circuit_buf_to_acir_format(std::move(command.circuit.bytecode)); + + // Create verification key based on whether it's standalone or not + std::vector vk_data; + if (command.standalone) { + // For standalone, we just need the circuit's verification key (not the full IVC VK) + acir_format::AcirProgram program{ constraint_system, /*witness=*/{} }; + std::shared_ptr proving_key = + get_acir_program_decider_proving_key(request, program); + auto verification_key = std::make_shared(proving_key->proving_key); + vk_data = to_buffer(*verification_key); + info("ClientIvcComputeVk - standalone VK derived, size: ", vk_data.size(), " bytes"); + } else { + vk_data = to_buffer(compute_vk_for_ivc(request, constraint_system.public_inputs.size())); + info("ClientIvcComputeVk - full IVC VK derived, size: ", vk_data.size(), " bytes"); + } + + return ClientIvcComputeVk::Response{ .verification_key = vk_data, .error_message = "" }; +} + +inline ClientIvcCheckPrecomputedVk::Response execute(BBRpcRequest& request, ClientIvcCheckPrecomputedVk&& command) +{ + (void)request; + (void)command; + throw_or_abort("code in progress! should not be called"); +} + +/** + * @brief Executes a command by visiting a variant of all possible commands. + * + * @param command The command to execute, consumed by this function. + * @param request The circuit registry (acting as the request context). + * @return A variant of all possible command responses. + */ +inline CommandResponse execute(BBRpcRequest& request, Command&& command) +{ + return std::move(command).visit( + [&request](auto&& cmd) -> CommandResponse { return execute(request, std::forward(cmd)); }); +} + +template typename T::Response execute_or_throw(BBRpcRequest& request, T&& command) +{ + auto response = execute(request, std::forward(command)); + if (!response.error_message.empty()) { + throw_or_abort(response.error_message); + } + return response; +} + +// Can only be called from the execution thread (the same as the main thread, except in threaded WASM). +inline std::vector execute_request(BBRpcRequest&& request, std::vector&& commands) +{ + std::vector responses; + responses.reserve(commands.size()); + for (Command& command : commands) { + responses.push_back(execute(request, std::move(command))); + if (!get_error_message(responses.back()).empty()) { + // If there was an error, we stop processing further commands. + break; + } + } + return responses; +} + +} // namespace bb::bbrpc diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/acir_bincode_mocks.hpp b/barretenberg/cpp/src/barretenberg/client_ivc/acir_bincode_mocks.hpp new file mode 100644 index 000000000000..dfcc40daf7f9 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/client_ivc/acir_bincode_mocks.hpp @@ -0,0 +1,126 @@ +#pragma once + +#include "barretenberg/dsl/acir_format/acir_format.hpp" +#include "barretenberg/dsl/acir_format/serde/witness_stack.hpp" +#include +#include + +namespace bb::acir_bincode_mocks { + +const size_t BIT_COUNT = 254; + +/** + * @brief Helper function to create a minimal circuit bytecode and witness for testing + * @return A pair of (circuit_bytecode, witness_data) + * + * The circuit implements: w0 * w1 = w2 + * Example witness: w0=2, w1=3, w2=6 (so 2*3=6) + */ +inline std::pair, std::vector> create_simple_circuit_bytecode() +{ + Acir::Circuit circuit; + + // No public inputs + circuit.public_parameters = Acir::PublicInputs{ {} }; + + std::string one = "0000000000000000000000000000000000000000000000000000000000000001"; + std::string minus_one = "30644e72e131a029b85045b68181585d2833e84879b9709143e1f593f0000000"; + + Acir::Expression expr; + + // Create constraint: w0 * w1 - w2 = 0 + expr.mul_terms = { { one, Acir::Witness{ 0 }, Acir::Witness{ 1 } } }; // w0 * w1 + expr.linear_combinations = { { minus_one, Acir::Witness{ 2 } } }; // -1 * w2 + expr.q_c = "0000000000000000000000000000000000000000000000000000000000000000"; + + Acir::Opcode::AssertZero assert_zero; + assert_zero.value = expr; + Acir::Opcode opcode; + opcode.value = assert_zero; + circuit.opcodes.push_back(opcode); + + circuit.current_witness_index = 3; + circuit.expression_width = Acir::ExpressionWidth{ Acir::ExpressionWidth::Unbounded{} }; + circuit.private_parameters = {}; + circuit.return_values = Acir::PublicInputs{ {} }; + circuit.assert_messages = {}; + + // Create the program + Acir::Program program; + program.functions = { circuit }; + program.unconstrained_functions = {}; + + // Create witness data + Witnesses::WitnessStack witness_stack; + Witnesses::StackItem stack_item{}; + + // w0=2, w1=3, w2=6 (so 2*3=6) + stack_item.witness.value = { + { Witnesses::Witness{ 0 }, "0000000000000000000000000000000000000000000000000000000000000002" }, // w0 = 2 + { Witnesses::Witness{ 1 }, "0000000000000000000000000000000000000000000000000000000000000003" }, // w1 = 3 + { Witnesses::Witness{ 2 }, "0000000000000000000000000000000000000000000000000000000000000006" } // w2 = 6 + }; + witness_stack.stack.push_back(stack_item); + + return { program.bincodeSerialize(), witness_stack.bincodeSerialize() }; +} + +/** + * @brief Create a simple kernel circuit for IVC testing + * + * @return Serialized kernel bytecode + */ +inline std::vector create_simple_kernel(size_t vk_size, bool is_init_kernel) +{ + Acir::Circuit circuit; + // Create witnesses equal to size of a mega VK in fields. + std::vector vk_inputs; + for (uint32_t i = 0; i < vk_size; i++) { + Acir::FunctionInput input{ { Acir::ConstantOrWitnessEnum::Witness{ i } }, BIT_COUNT }; + vk_inputs.push_back(input); + } + + // Modeled after noir-projects/mock-protocol-circuits/crates/mock-private-kernel-init/src/main.nr + // We mock the init or tail kernels using OINK or PG respectively. + Acir::BlackBoxFuncCall::RecursiveAggregation recursion{ + .verification_key = vk_inputs, + .proof = {}, + .public_inputs = {}, + // NOTE: If this starts failing after key hash becomes required need to pass as witness! (possibly after the VK, + // adding +1 to witness index below) + .key_hash = Acir::FunctionInput{ { Acir::ConstantOrWitnessEnum::Witness{ Acir::Witness{ 0 } } }, BIT_COUNT }, + .proof_type = is_init_kernel ? acir_format::PROOF_TYPE::OINK : acir_format::PROOF_TYPE::PG + }; + + Acir::BlackBoxFuncCall black_box_call; + black_box_call.value = recursion; + + circuit.opcodes.push_back(Acir::Opcode{ Acir::Opcode::BlackBoxFuncCall{ black_box_call } }); + circuit.current_witness_index = static_cast(vk_inputs.size()); + circuit.expression_width = Acir::ExpressionWidth{ Acir::ExpressionWidth::Bounded{ 3 } }; + + // Create the program with the circuit + Acir::Program program; + program.functions = { circuit }; + // Serialize the program using bincode + return program.bincodeSerialize(); +} + +/** + * @brief Create a kernel witness for IVC testing + * @param app_vk_fields The application verification key fields to include in witness + * @return Serialized witness data + */ +inline std::vector create_kernel_witness(const std::vector& app_vk_fields) +{ + Witnesses::WitnessStack kernel_witness; + kernel_witness.stack.push_back({}); + for (uint32_t i = 0; i < app_vk_fields.size(); i++) { + std::stringstream ss; + ss << app_vk_fields[i]; + kernel_witness.stack.back().witness.value[Witnesses::Witness{ i }] = ss.str(); + } + return kernel_witness.bincodeSerialize(); +} + +} // namespace bb::acir_bincode_mocks diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp b/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp index c7d8a39d0d22..5ddc1e7209b2 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/mock_circuit_producer.hpp @@ -185,7 +185,7 @@ class ClientIVCMockCircuitProducer { static ClientCircuit create_mock_circuit(ClientIVC& ivc, size_t log2_num_gates = 16) { ClientCircuit circuit{ ivc.goblin.op_queue }; - MockCircuits::construct_arithmetic_circuit(circuit, log2_num_gates); + MockCircuits::construct_arithmetic_circuit(circuit, log2_num_gates, /* include_public_inputs= */ false); return circuit; } diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.cpp index 2cbdb61ebebc..8575405795fb 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.cpp @@ -5,6 +5,27 @@ namespace bb { +std::vector compress(const std::vector& input) +{ + auto compressor = + std::unique_ptr{ libdeflate_alloc_compressor(6), + libdeflate_free_compressor }; + + // Worst case size for gzip compression + size_t max_compressed_size = libdeflate_gzip_compress_bound(compressor.get(), input.size()); + std::vector compressed(max_compressed_size); + + size_t actual_compressed_size = + libdeflate_gzip_compress(compressor.get(), input.data(), input.size(), compressed.data(), compressed.size()); + + if (actual_compressed_size == 0) { + THROW std::runtime_error("Failed to compress data"); + } + + compressed.resize(actual_compressed_size); + return compressed; +} + std::vector decompress(const void* bytes, size_t size) { std::vector content; @@ -52,12 +73,26 @@ template T unpack_from_file(const std::filesystem::path& filename) return result; } +// TODO(#7371) we should not have so many levels of serialization here. +std::vector PrivateExecutionStepRaw::load(const std::filesystem::path& input_path) +{ + PROFILE_THIS(); + return unpack_from_file>(input_path); +} + +// TODO(#7371) we should not have so many levels of serialization here. +void PrivateExecutionStepRaw::self_decompress() +{ + bytecode = decompress(bytecode.data(), bytecode.size()); + witness = decompress(witness.data(), witness.size()); +} + // TODO(#7371) we should not have so many levels of serialization here. std::vector PrivateExecutionStepRaw::load_and_decompress( const std::filesystem::path& input_path) { PROFILE_THIS(); - auto raw_steps = unpack_from_file>(input_path); + auto raw_steps = load(input_path); for (PrivateExecutionStepRaw& step : raw_steps) { step.bytecode = decompress(step.bytecode.data(), step.bytecode.size()); step.witness = decompress(step.witness.data(), step.witness.size()); @@ -131,4 +166,29 @@ std::shared_ptr PrivateExecutionSteps::accumulate() return ivc; } + +void PrivateExecutionStepRaw::compress_and_save(std::vector&& steps, + const std::filesystem::path& output_path) +{ + // First, compress the bytecode and witness fields of each step + for (PrivateExecutionStepRaw& step : steps) { + step.bytecode = compress(step.bytecode); + step.witness = compress(step.witness); + step.vk = step.vk; + step.function_name = step.function_name; + } + + // Serialize to msgpack + std::stringstream ss; + msgpack::pack(ss, steps); + std::string packed_data = ss.str(); + + // Write to file + std::ofstream file(output_path, std::ios::binary); + if (!file) { + THROW std::runtime_error("Failed to open file for writing: " + output_path.string()); + } + file.write(packed_data.data(), static_cast(packed_data.size())); + file.close(); +} } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.hpp b/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.hpp index 89c68869952b..3c372fa449cf 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.hpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.hpp @@ -32,8 +32,12 @@ struct PrivateExecutionStepRaw { // Unrolled from MSGPACK_FIELDS for custom name for function_name. void msgpack(auto pack_fn) { pack_fn(NVP(bytecode, witness, vk), "functionName", function_name); }; + void self_decompress(); static std::vector load_and_decompress(const std::filesystem::path& input_path); + static std::vector load(const std::filesystem::path& input_path); static std::vector parse_uncompressed(const std::vector& buf); + static void compress_and_save(std::vector&& steps, + const std::filesystem::path& output_path); }; // TODO(https://github.com/AztecProtocol/barretenberg/issues/1162) this should have a common code path with diff --git a/barretenberg/cpp/src/barretenberg/common/named_union.hpp b/barretenberg/cpp/src/barretenberg/common/named_union.hpp new file mode 100644 index 000000000000..acb2196c3140 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/common/named_union.hpp @@ -0,0 +1,155 @@ +#pragma once +#include "barretenberg/common/throw_or_abort.hpp" +#include "barretenberg/serialize/msgpack.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace bb { + +/** + * @brief Concept to check if a type has a static NAME member + */ +template +concept HasName = requires { + { + T::NAME + } -> std::convertible_to; +}; + +/** + * @brief A wrapper around std::variant that provides msgpack serialization based on type names + * + * Each type in the variant must have a static constexpr NAME member that identifies it. + * During serialization, the NAME is written first, then the object. + * During deserialization, the NAME is read first to determine which type to construct. + */ +template class NamedUnion { + public: + using VariantType = std::variant; + + private: + VariantType value_; + + // Helper to get index from type name + template static std::optional get_index_from_name(std::string_view name) + { + if constexpr (I < sizeof...(Types)) { + using CurrentType = std::variant_alternative_t; + if (name == CurrentType::NAME) { + return I; + } + return get_index_from_name(name); + } + return std::nullopt; + } + + // Helper to construct variant by index + template static VariantType construct_by_index(size_t index, auto& o) + { + if constexpr (I < sizeof...(Types)) { + if (I == index) { + using CurrentType = std::variant_alternative_t; + CurrentType obj; + o.convert(obj); + return obj; + } + return construct_by_index(index, o); + } + throw_or_abort("Invalid variant index"); + } + + public: + NamedUnion() = default; + + template + requires(std::is_constructible_v) + // NOLINTNEXTLINE(bugprone-forwarding-reference-overload) + NamedUnion(T&& t) + : value_(std::forward(t)) + {} + + // Conversion operator to get the underlying variant + operator VariantType&() { return value_; } + operator const VariantType&() const { return value_; } + + // Access the underlying variant + VariantType& get() { return value_; } + const VariantType& get() const { return value_; } + + // Visit the variant + template decltype(auto) visit(Visitor&& vis) && + { + return std::visit(std::forward(vis), std::move(value_)); + } + + template decltype(auto) visit(Visitor&& vis) const& + { + return std::visit(std::forward(vis), value_); + } + + // Get the current type name + std::string_view get_type_name() const + { + return std::visit([](const auto& obj) -> std::string_view { return std::decay_t::NAME; }, + value_); + } + + // Msgpack serialization + void msgpack_pack(auto& packer) const + { + packer.pack_array(2); + // First pack the type name + std::string_view type_name = get_type_name(); + packer.pack(type_name); + + // Then pack the actual object + std::visit([&packer](const auto& obj) { packer.pack(obj); }, value_); + } + + // Msgpack deserialization + void msgpack_unpack(auto o) + { + // access object assuming it is an array of size 2 + if (!o.is_array() || o.via.array.size != 2) { + throw_or_abort("Expected an array of size 2 for NamedUnion deserialization"); + } + auto& arr = o.via.array; + if (!arr.ptr[0].is_string()) { + throw_or_abort("Expected first element to be a string (type name) in NamedUnion deserialization"); + } + std::string_view type_name = arr.ptr[0].template as(); + auto index_opt = get_index_from_name(type_name); + if (!index_opt.has_value()) { + throw_or_abort("Unknown type name in NamedUnion deserialization: " + std::string(type_name)); + } + size_t index = index_opt.value(); + // Now construct the variant using the index + value_ = construct_by_index(index, arr.ptr[1]); + } + + // Msgpack schema + void msgpack_schema(auto& packer) const + { + packer.pack_array(2); + packer.pack("named_union"); + packer.pack_array(sizeof...(Types)); + ( + [&packer, this]() { + packer.pack_array(2); + packer.pack(Types::NAME); + // Abitrary mutable object. + packer.pack_schema(*std::make_unique()); + }(), + ...); /* pack schemas of all template Args */ + } +}; + +// Deduction guide +template NamedUnion(std::variant) -> NamedUnion; + +} // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp index 3acac026dcd8..80156d323cfa 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/mock_circuits.hpp @@ -135,14 +135,16 @@ class MockCircuits { * @param num_gates */ template - static void construct_arithmetic_circuit(Builder& builder, const size_t target_log2_dyadic_size = 4) + static void construct_arithmetic_circuit(Builder& builder, + const size_t target_log2_dyadic_size = 4, + bool include_public_inputs = true) { const size_t target_dyadic_size = 1 << target_log2_dyadic_size; const size_t num_preamble_gates = builder.num_gates; ASSERT(target_dyadic_size >= num_preamble_gates); // For good measure, include a gate with some public inputs - if (target_dyadic_size > num_preamble_gates) { + if (include_public_inputs && target_dyadic_size > num_preamble_gates) { add_arithmetic_gates_with_public_inputs(builder, 1); }