diff --git a/.github/workflows/auto-rebase-pr.yml b/.github/workflows/auto-rebase-pr.yml index 75e310e83150..96371148ea93 100644 --- a/.github/workflows/auto-rebase-pr.yml +++ b/.github/workflows/auto-rebase-pr.yml @@ -30,9 +30,18 @@ jobs: env: GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} run: | + set -x pr_head_ref=${{ github.event.pull_request.head.ref }} pr_base_ref=${{ github.event.pull_request.base.ref }} - ./scripts/auto_rebase_pr.sh "$pr_head_ref" "$pr_base_ref" + # ensure we have up to date head/base + git fetch origin "$pr_base_ref" "$pr_head_ref" + # Be sure to run the auto rebase script such that we don't write to it while it's being read. + cp ./scripts/auto_rebase_pr.sh .github/temp-script.sh + bash .github/temp-script.sh "$pr_head_ref" "$pr_base_ref" + rebase_sha=$(git rev-parse HEAD) + git switch --force-create "$pr_head_ref" "origin/$pr_base_ref" + git reset --hard "$rebase_sha" + git push origin "$pr_head_ref" --force-with-lease - name: Post failure comment if: failure() || steps.rebase.outcome == 'failure' diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index dd48173c92f2..9fb2842b64dc 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -308,11 +308,9 @@ case "$cmd" in # Download cached flow inputs from the specified commit export AZTEC_CACHE_COMMIT=$commit_hash + # TODO currently does nothing! to reinstate in cache_download export FORCE_CACHE_DOWNLOAD=${FORCE_CACHE_DOWNLOAD:-1} - echo "Running with FORCE_CACHE_DOWNLOAD=1. This should work, but as a workaround you can set FORCE_CACHE_DOWNLOAD=0 before the bench_ivc call to build some parts." - ../../noir/bootstrap.sh - USE_CIRCUITS_CACHE=1 ../../noir-projects/noir-protocol-circuits/bootstrap.sh - yarn --cwd ../../yarn-project/bb-prover generate + USE_CIRCUITS_CACHE=1 BOOTSTRAP_AFTER=barretenberg BOOSTRAP_TO=yarn-project ../../bootstrap.sh rm -rf bench-out diff --git a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp index 9bf60226321f..5b4add39168d 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp +++ b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp @@ -1,5 +1,7 @@ #include "api_client_ivc.hpp" +#include "barretenberg/api/bbapi.hpp" #include "barretenberg/api/file_io.hpp" +#include "barretenberg/api/get_bytecode.hpp" #include "barretenberg/api/log.hpp" #include "barretenberg/api/write_prover_output.hpp" #include "barretenberg/client_ivc/client_ivc.hpp" @@ -13,37 +15,11 @@ #include "barretenberg/serialize/msgpack.hpp" #include "barretenberg/serialize/msgpack_check_eq.hpp" #include +#include #include namespace bb { - -acir_format::WitnessVector witness_map_to_witness_vector(std::map const& witness_map) -{ - acir_format::WitnessVector wv; - size_t index = 0; - for (auto& e : witness_map) { - uint64_t value = stoull(e.first); - // ACIR uses a sparse format for WitnessMap where unused witness indices may be left unassigned. - // To ensure that witnesses sit at the correct indices in the `WitnessVector`, we fill any indices - // which do not exist within the `WitnessMap` with the dummy value of zero. - while (index < value) { - wv.push_back(fr(0)); - index++; - } - wv.push_back(fr(uint256_t(e.second))); - index++; - } - return wv; -} - -std::shared_ptr get_acir_program_decider_proving_key(acir_format::AcirProgram& program) -{ - ClientIVC::ClientCircuit builder = acir_format::create_circuit(program); - - // Construct the verification key via the prover-constructed proving key with the proper trace settings - TraceSettings trace_settings{ AZTEC_TRACE_STRUCTURE }; - return std::make_shared(builder, trace_settings); -} +namespace { // anonymous namespace /** * @brief Compute and write to file a MegaHonk VK for a circuit to be accumulated in the IVC @@ -53,52 +29,57 @@ std::shared_ptr get_acir_program_decider_proving_k * @param bytecode_path * @param witness_path */ -void write_standalone_vk(const std::string& output_data_type, - const std::string& bytecode_path, - const std::string& output_path) +void write_standalone_vk(const std::string& output_format, + const std::filesystem::path& bytecode_path, + const std::filesystem::path& output_path) { - - acir_format::AcirProgram program{ get_constraint_system(bytecode_path), /*witness=*/{} }; - std::shared_ptr proving_key = get_acir_program_decider_proving_key(program); - auto verification_key = std::make_shared(proving_key->get_precomputed()); - PubInputsProofAndKey to_write{ .key = verification_key }; - - write(to_write, output_data_type, "vk", output_path); -} - -size_t get_num_public_inputs_in_circuit(const std::filesystem::path& bytecode_path) -{ - using namespace acir_format; - acir_format::AcirProgram program{ get_constraint_system(bytecode_path), /*witness=*/{} }; - return program.constraints.public_inputs.size(); + auto bytecode = get_bytecode(bytecode_path); + auto response = bbapi::ClientIvcComputeStandaloneVk{ + .circuit = { .name = "standalone_circuit", .bytecode = std::move(bytecode) } + }.execute(); + + bool wrote_file = false; + bool is_stdout = output_path == "-"; + auto write_fn = [&](const std::filesystem::path& path, const auto& data) { + if (is_stdout) { + write_bytes_to_stdout(data); + } else { + write_file(path, data); + } + }; + if (output_format == "bytes_and_fields" && is_stdout) { + throw_or_abort("Cannot write to stdout in bytes_and_fields format."); + } + if (output_format == "bytes" || output_format == "bytes_and_fields") { + write_fn(output_path / "vk", response.bytes); + wrote_file = true; + } + if (output_format == "fields" || output_format == "bytes_and_fields") { + std::string json = field_elements_to_json(response.fields); + write_fn(output_path / "vk_fields.json", std::vector(json.begin(), json.end())); + wrote_file = true; + } + if (!wrote_file) { + throw_or_abort("Unsupported output format for standalone vk: " + output_format); + } } -void write_vk_for_ivc(const std::string& output_format, - size_t num_public_inputs_in_final_circuit, - const std::filesystem::path& output_dir) +void write_civc_vk(const std::string& output_format, + size_t num_public_inputs_in_final_circuit, + const std::filesystem::path& output_dir) { if (output_format != "bytes") { throw_or_abort("Unsupported output format for ClientIVC vk: " + output_format); } - ClientIVC ivc{ /*num_circuits=*/2, { AZTEC_TRACE_STRUCTURE } }; - ClientIVCMockCircuitProducer circuit_producer; - - // Initialize the IVC with an arbitrary circuit - // We segfault if we only call accumulate once - static constexpr size_t SMALL_ARBITRARY_LOG_CIRCUIT_SIZE{ 5 }; - MegaCircuitBuilder circuit_0 = circuit_producer.create_next_circuit(ivc, SMALL_ARBITRARY_LOG_CIRCUIT_SIZE); - ivc.accumulate(circuit_0); - - // Create another circuit and accumulate - MegaCircuitBuilder circuit_1 = - circuit_producer.create_next_circuit(ivc, SMALL_ARBITRARY_LOG_CIRCUIT_SIZE, num_public_inputs_in_final_circuit); - ivc.accumulate(circuit_1); - // Construct the hiding circuit and its VK (stored internally in the IVC) - ivc.construct_hiding_circuit_key(); + // Since we need to specify the number of public inputs but ClientIvcComputeIvcVk derives it from bytecode, + // we need to create a mock circuit with the correct number of public inputs + // For now, we'll use the compute_civc_vk function directly as it was designed for this purpose + bbapi::BBApiRequest request; + auto vk = bbapi::compute_civc_vk(request, num_public_inputs_in_final_circuit); + const auto buf = to_buffer(vk); const bool output_to_stdout = output_dir == "-"; - const auto buf = to_buffer(ivc.get_vk()); if (output_to_stdout) { write_bytes_to_stdout(buf); @@ -107,33 +88,53 @@ void write_vk_for_ivc(const std::string& output_format, } } -void write_vk_for_ivc(const std::string& output_data_type, - const std::string& bytecode_path, - const std::filesystem::path& output_dir) +void write_civc_vk(const std::string& output_data_type, + const std::string& bytecode_path, + const std::filesystem::path& output_dir) { - const size_t num_public_inputs_in_final_circuit = get_num_public_inputs_in_circuit(bytecode_path); - info("num_public_inputs_in_final_circuit: ", num_public_inputs_in_final_circuit); - write_vk_for_ivc(output_data_type, num_public_inputs_in_final_circuit, output_dir); + if (output_data_type != "bytes") { + throw_or_abort("Unsupported output format for ClientIVC vk: " + output_data_type); + } + + auto bytecode = get_bytecode(bytecode_path); + + auto response = bbapi::ClientIvcComputeIvcVk{ + .circuit = { .name = "final_circuit", .bytecode = std::move(bytecode) } + }.execute(); + + const bool output_to_stdout = output_dir == "-"; + if (output_to_stdout) { + write_bytes_to_stdout(response.bytes); + } else { + write_file(output_dir / "vk", response.bytes); + } } +} // anonymous namespace void ClientIVCAPI::prove(const Flags& flags, const std::filesystem::path& input_path, const std::filesystem::path& output_dir) { - PrivateExecutionSteps steps; - steps.parse(PrivateExecutionStepRaw::load_and_decompress(input_path)); + bbapi::BBApiRequest request; + std::vector raw_steps = PrivateExecutionStepRaw::load_and_decompress(input_path); - std::shared_ptr ivc = steps.accumulate(); - ClientIVC::Proof proof = ivc->prove(); + bbapi::ClientIvcStart{ .num_circuits = raw_steps.size() }.execute(request); + + size_t loaded_circuit_public_inputs_size = 0; + for (const auto& step : raw_steps) { + bbapi::ClientIvcLoad{ + .circuit = { .name = step.function_name, .bytecode = step.bytecode, .verification_key = step.vk } + }.execute(request); - // We verify this proof. Another bb call to verify has the overhead of loading the SRS, - // and it is mysterious if this transaction fails later in the lifecycle. - // The files are still written in case they are needed to investigate this failure. - if (!ivc->verify(proof)) { - THROW std::runtime_error("Failed to verify the private (ClientIVC) transaction proof!"); + // NOLINTNEXTLINE(bugprone-unchecked-optional-access): we know the optional has been set here. + loaded_circuit_public_inputs_size = request.loaded_circuit_constraints->public_inputs.size(); + info("ClientIVC: accumulating " + step.function_name); + bbapi::ClientIvcAccumulate{ .witness = step.witness }.execute(request); } + auto proof = bbapi::ClientIvcProve{}.execute(request).proof; + // We'd like to use the `write` function that UltraHonkAPI uses, but there are missing functions for creating // std::string representations of vks that don't feel worth implementing const bool output_to_stdout = output_dir == "-"; @@ -153,8 +154,7 @@ void ClientIVCAPI::prove(const Flags& flags, if (flags.write_vk) { vinfo("writing ClientIVC vk in directory ", output_dir); - const size_t num_public_inputs_in_final_circuit = steps.folding_stack.back().constraints.public_inputs.size(); - write_vk_for_ivc("bytes", num_public_inputs_in_final_circuit, output_dir); + write_civc_vk("bytes", loaded_circuit_public_inputs_size, output_dir); } } @@ -194,21 +194,22 @@ void ClientIVCAPI::write_solidity_verifier([[maybe_unused]] const Flags& flags, } bool ClientIVCAPI::check_precomputed_vks(const std::filesystem::path& input_path) - { - PrivateExecutionSteps steps; - steps.parse(PrivateExecutionStepRaw::load_and_decompress(input_path)); + bbapi::BBApiRequest request; + std::vector raw_steps = PrivateExecutionStepRaw::load_and_decompress(input_path); - for (auto [program, precomputed_vk, function_name] : - zip_view(steps.folding_stack, steps.precomputed_vks, steps.function_names)) { - if (precomputed_vk == nullptr) { - info("FAIL: Expected precomputed vk for function ", function_name); + for (const auto& step : raw_steps) { + if (step.vk.empty()) { + info("FAIL: Expected precomputed vk for function ", step.function_name); return false; } - std::shared_ptr proving_key = get_acir_program_decider_proving_key(program); - auto computed_vk = std::make_shared(proving_key->get_precomputed()); - std::string error_message = "FAIL: Precomputed vk does not match computed vk for function " + function_name; - if (!msgpack::msgpack_check_eq(*computed_vk, *precomputed_vk, error_message)) { + auto response = bbapi::ClientIvcCheckPrecomputedVk{ .circuit = { .name = step.function_name, + .bytecode = step.bytecode, + .verification_key = step.vk }, + .function_name = step.function_name } + .execute(); + + if (!response.valid) { return false; } } @@ -221,7 +222,7 @@ void ClientIVCAPI::write_vk(const Flags& flags, { if (flags.verifier_type == "ivc") { - write_vk_for_ivc(flags.output_format, bytecode_path, output_path); + write_civc_vk(flags.output_format, bytecode_path, output_path); } else if (flags.verifier_type == "standalone") { write_standalone_vk(flags.output_format, bytecode_path, output_path); } else { diff --git a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.hpp b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.hpp index b9d74abb52b9..0e568ec0706e 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.hpp +++ b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.hpp @@ -44,8 +44,6 @@ void gate_count_for_ivc(const std::string& bytecode_path, bool include_gates_per void write_arbitrary_valid_client_ivc_proof_and_vk_to_file(const std::filesystem::path& output_dir); -acir_format::WitnessVector witness_map_to_witness_vector(std::map const& witness_map); - std::vector decompress(const void* bytes, size_t size); } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp index 6b2f4d97d6ae..12af7213f0be 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp +++ b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp @@ -1,6 +1,6 @@ #include "api_client_ivc.hpp" -#include "barretenberg/api/bbrpc_commands.hpp" -#include "barretenberg/api/bbrpc_execute.hpp" +#include "barretenberg/api/bbapi.hpp" +#include "barretenberg/api/bbapi_execute.hpp" #include "barretenberg/api/file_io.hpp" #include "barretenberg/client_ivc/acir_bincode_mocks.hpp" #include "barretenberg/client_ivc/client_ivc.hpp" @@ -14,6 +14,7 @@ #include "barretenberg/dsl/acir_format/serde/witness_stack.hpp" #include "barretenberg/flavor/mega_flavor.hpp" #include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/serialize/msgpack_check_eq.hpp" #include "barretenberg/srs/global_crs.hpp" #include #include @@ -44,24 +45,21 @@ void create_test_private_execution_steps(const std::filesystem::path& output_pat auto [app_bytecode, app_witness_data] = acir_bincode_mocks::create_simple_circuit_bytecode(); // Get the VK for the app circuit - bbrpc::BBRpcRequest request; + bbapi::BBApiRequest request; - auto app_vk = - bbrpc::execute(request, - bbrpc::ClientIvcComputeVk{ .circuit = { .name = "app_circuit", .bytecode = app_bytecode }, - .standalone = true }) - .verification_key; + auto app_vk_response = + bbapi::ClientIvcComputeStandaloneVk{ .circuit = { .name = "app_circuit", .bytecode = app_bytecode } }.execute(); + auto app_vk = app_vk_response.bytes; auto app_vk_fields = from_buffer(app_vk).to_field_elements(); // Now create a kernel circuit that verifies the app circuit auto kernel_bytecode = acir_bincode_mocks::create_simple_kernel(app_vk_fields.size(), /*is_init_kernel=*/true); auto kernel_witness_data = acir_bincode_mocks::create_kernel_witness(app_vk_fields); - auto kernel_vk = - bbrpc::execute(request, - bbrpc::ClientIvcComputeVk{ .circuit = { .name = "kernel_circuit", .bytecode = kernel_bytecode }, - .standalone = true }) - .verification_key; + auto kernel_vk_response = bbapi::ClientIvcComputeStandaloneVk{ + .circuit = { .name = "kernel_circuit", .bytecode = kernel_bytecode } + }.execute(); + auto kernel_vk = kernel_vk_response.bytes; // Create PrivateExecutionStepRaw for the kernel std::vector raw_steps; @@ -103,13 +101,10 @@ std::vector compress(const std::vector& input); ClientIVC::MegaVerificationKey get_ivc_vk(const std::filesystem::path& test_dir) { auto [app_bytecode, app_witness_data] = acir_bincode_mocks::create_simple_circuit_bytecode(); - bbrpc::BBRpcRequest request; - // First create an app standalone VK. - auto app_vk = - bbrpc::execute(request, - bbrpc::ClientIvcComputeVk{ .circuit = { .name = "app_circuit", .bytecode = app_bytecode }, - .standalone = true }) - .verification_key; + bbapi::BBApiRequest request; + auto app_vk_response = + bbapi::ClientIvcComputeStandaloneVk{ .circuit = { .name = "app_circuit", .bytecode = app_bytecode } }.execute(); + auto app_vk = app_vk_response.bytes; auto app_vk_fields = from_buffer(app_vk).to_field_elements(); // Use this to get the size of the vk. auto bytecode = acir_bincode_mocks::create_simple_kernel(app_vk_fields.size(), /*is_init_kernel=*/false); @@ -173,7 +168,7 @@ TEST_F(ClientIVCAPITests, ProveAndVerifyFileBasedFlow) EXPECT_TRUE(verify_proof()); } -// WORKTODO(bbrpc): Expand on this. +// WORKTODO(bbapi): Expand on this. TEST_F(ClientIVCAPITests, WriteVkFieldsSmokeTest) { // Create a simple circuit bytecode @@ -267,23 +262,17 @@ TEST_F(ClientIVCAPITests, CheckPrecomputedVksMismatch) // Create a simple circuit auto [bytecode, witness_data] = acir_bincode_mocks::create_simple_circuit_bytecode(); - bbrpc::BBRpcRequest request; - size_t vk_size = - from_buffer( - bbrpc::execute(request, - bbrpc::ClientIvcComputeVk{ .circuit = { .name = "simple_circuit", .bytecode = bytecode }, - .standalone = true }) - .verification_key) - .to_field_elements() - .size(); + bbapi::BBApiRequest request; + auto vk_response = + bbapi::ClientIvcComputeStandaloneVk{ .circuit = { .name = "simple_circuit", .bytecode = bytecode } }.execute(); + size_t vk_size = from_buffer(vk_response.bytes).to_field_elements().size(); // Create a WRONG verification key (use a different circuit) auto different_bytecode = acir_bincode_mocks::create_simple_kernel(vk_size, /*is_init_kernel=*/true); - auto vk = bbrpc::execute( - request, - bbrpc::ClientIvcComputeVk{ .circuit = { .name = "different_circuit", .bytecode = different_bytecode }, - .standalone = true }) - .verification_key; + auto vk_response2 = bbapi::ClientIvcComputeStandaloneVk{ + .circuit = { .name = "different_circuit", .bytecode = different_bytecode } + }.execute(); + auto vk = vk_response2.bytes; // Create PrivateExecutionStepRaw with wrong VK std::vector raw_steps; diff --git a/barretenberg/cpp/src/barretenberg/api/bbapi.hpp b/barretenberg/cpp/src/barretenberg/api/bbapi.hpp new file mode 100644 index 000000000000..1d825e660fc3 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/api/bbapi.hpp @@ -0,0 +1,13 @@ +#pragma once +/** + * @file bbapi_commands.hpp + * @brief Central command definitions BB's outward-facing API. + * + * This file includes and exports all command structures from specialized headers + * and provides unified Command and CommandResponse types for the API. + */ +#include "barretenberg/api/bbapi_client_ivc.hpp" +#include "barretenberg/api/bbapi_execute.hpp" +#include "barretenberg/api/bbapi_shared.hpp" +#include "barretenberg/api/bbapi_ultra_honk.hpp" +#include "barretenberg/common/named_union.hpp" diff --git a/barretenberg/cpp/src/barretenberg/api/bbapi_client_ivc.cpp b/barretenberg/cpp/src/barretenberg/api/bbapi_client_ivc.cpp new file mode 100644 index 000000000000..863d32fb0697 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/api/bbapi_client_ivc.cpp @@ -0,0 +1,184 @@ +#include "barretenberg/api/bbapi_client_ivc.hpp" +#include "barretenberg/client_ivc/mock_circuit_producer.hpp" +#include "barretenberg/common/log.hpp" +#include "barretenberg/common/throw_or_abort.hpp" +#include "barretenberg/dsl/acir_format/acir_format.hpp" +#include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp" +#include "barretenberg/dsl/acir_format/serde/witness_stack.hpp" +#include "barretenberg/serialize/msgpack_check_eq.hpp" +#include "barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp" + +namespace bb::bbapi { + +ClientIvcStart::Response ClientIvcStart::execute(BBApiRequest& request) && +{ + request.ivc_in_progress = std::make_shared(num_circuits, request.trace_settings); + request.ivc_stack_depth = 0; + return Response{}; +} + +ClientIvcLoad::Response ClientIvcLoad::execute(BBApiRequest& request) && +{ + if (!request.ivc_in_progress) { + throw_or_abort("ClientIVC not started. Call ClientIvcStart first."); + } + + request.loaded_circuit_name = circuit.name; + request.loaded_circuit_constraints = acir_format::circuit_buf_to_acir_format(std::move(circuit.bytecode)); + request.loaded_circuit_vk = circuit.verification_key; + + info("ClientIvcLoad - loaded circuit '", request.loaded_circuit_name, "'"); + + return Response{}; +} + +ClientIvcAccumulate::Response ClientIvcAccumulate::execute(BBApiRequest& request) && +{ + if (!request.ivc_in_progress) { + throw_or_abort("ClientIVC not started. Call ClientIvcStart first."); + } + + if (!request.loaded_circuit_constraints.has_value()) { + throw_or_abort("No circuit loaded. Call ClientIvcLoad first."); + } + + acir_format::WitnessVector witness_data = acir_format::witness_buf_to_witness_data(std::move(witness)); + acir_format::AcirProgram program{ std::move(request.loaded_circuit_constraints.value()), std::move(witness_data) }; + + const acir_format::ProgramMetadata metadata{ request.ivc_in_progress }; + auto circuit = acir_format::create_circuit(program, metadata); + + std::shared_ptr precomputed_vk; + if (!request.loaded_circuit_vk.empty()) { + precomputed_vk = from_buffer>(request.loaded_circuit_vk); + } + + info("ClientIvcAccumulate - accumulating circuit '", request.loaded_circuit_name, "'"); + request.ivc_in_progress->accumulate(circuit, precomputed_vk); + request.ivc_stack_depth++; + + request.loaded_circuit_constraints.reset(); + request.loaded_circuit_vk.clear(); + + return Response{}; +} + +ClientIvcProve::Response ClientIvcProve::execute(BBApiRequest& request) && +{ + if (!request.ivc_in_progress) { + throw_or_abort("ClientIVC not started. Call ClientIvcStart first."); + } + + if (request.ivc_stack_depth == 0) { + throw_or_abort("No circuits accumulated. Call ClientIvcAccumulate first."); + } + + info("ClientIvcProve - generating proof for ", request.ivc_stack_depth, " accumulated circuits"); + + ClientIVC::Proof proof = request.ivc_in_progress->prove(); + + // We verify this proof. Another bb call to verify has some overhead of loading VK/proof/SRS, + // and it is mysterious if this transaction fails later in the lifecycle. + if (!request.ivc_in_progress->verify(proof)) { + throw_or_abort("Failed to verify the generated proof!"); + } + + request.ivc_in_progress.reset(); + request.ivc_stack_depth = 0; + + Response response; + response.proof = std::move(proof); + return response; +} + +static std::shared_ptr get_acir_program_decider_proving_key( + const BBApiRequest& request, acir_format::AcirProgram& program) +{ + ClientIVC::ClientCircuit builder = acir_format::create_circuit(program); + + // Construct the verification key via the prover-constructed proving key with the proper trace settings + return std::make_shared(builder, request.trace_settings); +} + +ClientIVC::VerificationKey compute_civc_vk(const BBApiRequest& request, size_t num_public_inputs_in_final_circuit) +{ + ClientIVC ivc{ /* num_circuits */ 2, request.trace_settings }; + ClientIVCMockCircuitProducer circuit_producer; + + // Initialize the IVC with an arbitrary circuit + // We segfault if we only call accumulate once + static constexpr size_t SMALL_ARBITRARY_LOG_CIRCUIT_SIZE{ 5 }; + MegaCircuitBuilder circuit_0 = circuit_producer.create_next_circuit(ivc, SMALL_ARBITRARY_LOG_CIRCUIT_SIZE); + ivc.accumulate(circuit_0); + + // Create another circuit and accumulate + MegaCircuitBuilder circuit_1 = + circuit_producer.create_next_circuit(ivc, SMALL_ARBITRARY_LOG_CIRCUIT_SIZE, num_public_inputs_in_final_circuit); + ivc.accumulate(circuit_1); + + // Construct the hiding circuit and its VK (stored internally in the IVC) + ivc.construct_hiding_circuit_key(); + + return ivc.get_vk(); +} + +ClientIvcComputeStandaloneVk::Response ClientIvcComputeStandaloneVk::execute(BB_UNUSED const BBApiRequest& request) && +{ + info("ClientIvcComputeStandaloneVk - deriving VK for circuit '", circuit.name, "'"); + + auto constraint_system = acir_format::circuit_buf_to_acir_format(std::move(circuit.bytecode)); + + acir_format::AcirProgram program{ constraint_system, /*witness=*/{} }; + std::shared_ptr proving_key = get_acir_program_decider_proving_key(request, program); + auto verification_key = std::make_shared(proving_key->get_precomputed()); + + Response response; + response.bytes = to_buffer(*verification_key); + response.fields = verification_key->to_field_elements(); + + info("ClientIvcComputeStandaloneVk - VK derived, size: ", response.bytes.size(), " bytes"); + + return response; +} + +ClientIvcComputeIvcVk::Response ClientIvcComputeIvcVk::execute(const BBApiRequest& request) && +{ + info("ClientIvcComputeIvcVk - deriving IVC VK for circuit '", circuit.name, "'"); + + auto constraint_system = acir_format::circuit_buf_to_acir_format(std::move(circuit.bytecode)); + + auto vk = compute_civc_vk(request, constraint_system.public_inputs.size()); + + Response response; + response.bytes = to_buffer(vk); + + info("ClientIvcComputeIvcVk - IVC VK derived, size: ", response.bytes.size(), " bytes"); + + return response; +} + +ClientIvcCheckPrecomputedVk::Response ClientIvcCheckPrecomputedVk::execute(const BBApiRequest& request) && +{ + acir_format::AcirProgram program{ acir_format::circuit_buf_to_acir_format(std::move(circuit.bytecode)), + /*witness=*/{} }; + + std::shared_ptr proving_key = get_acir_program_decider_proving_key(request, program); + auto computed_vk = std::make_shared(proving_key->get_precomputed()); + + if (circuit.verification_key.empty()) { + info("FAIL: Expected precomputed vk for function ", function_name); + throw_or_abort("Missing precomputed VK"); + } + + auto precomputed_vk = from_buffer>(circuit.verification_key); + + Response response; + response.valid = true; + std::string error_message = "Precomputed vk does not match computed vk for function " + function_name; + if (!msgpack::msgpack_check_eq(*computed_vk, *precomputed_vk, error_message)) { + response.valid = false; + } + return response; +} + +} // namespace bb::bbapi diff --git a/barretenberg/cpp/src/barretenberg/api/bbapi_client_ivc.hpp b/barretenberg/cpp/src/barretenberg/api/bbapi_client_ivc.hpp new file mode 100644 index 000000000000..9f1500eec6f5 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/api/bbapi_client_ivc.hpp @@ -0,0 +1,182 @@ +#pragma once +/** + * @file bbapi_client_ivc.hpp + * @brief ClientIVC-specific command definitions for the Barretenberg RPC API. + * + * This file contains command structures for ClientIVC (Client-side Incrementally Verifiable Computation) + * operations including circuit loading, accumulation, proving, and verification key computation. + */ +#include "barretenberg/api/bbapi_shared.hpp" +#include "barretenberg/client_ivc/client_ivc.hpp" +#include "barretenberg/common/named_union.hpp" +#include "barretenberg/honk/proof_system/types/proof.hpp" +#include + +namespace bb::bbapi { + +/** + * @brief Helper function to compute verification key for IVC + * @param request The API request context + * @param num_public_inputs_in_final_circuit Number of public inputs in the final circuit + * @return The computed IVC verification key + */ +ClientIVC::VerificationKey compute_civc_vk(const BBApiRequest& request, size_t num_public_inputs_in_final_circuit); + +/** + * @struct ClientIvcStart + * @brief Initialize a new ClientIVC instance for incremental proof accumulation + * + * @note Only one IVC request can be made at a time for each batch_request. + */ +struct ClientIvcStart { + static constexpr const char* NAME = "ClientIvcStart"; + + /** + * @struct Response + * @brief Empty response indicating successful initialization + */ + struct Response { + static constexpr const char* NAME = "ClientIvcStartResponse"; + // Empty response - success indicated by no exception + }; + // Number of circuits to be accumulated. + size_t num_circuits; + Response execute(BBApiRequest& request) &&; +}; + +/** + * @struct ClientIvcLoad + * @brief Load a circuit into the ClientIVC instance for accumulation + */ +struct ClientIvcLoad { + static constexpr const char* NAME = "ClientIvcLoad"; + + /** + * @struct Response + * @brief Empty response indicating successful circuit loading + */ + struct Response { + static constexpr const char* NAME = "ClientIvcLoadResponse"; + // Empty response - success indicated by no exception + }; + + /** @brief Circuit to be loaded with its bytecode and verification key */ + CircuitInput circuit; + Response execute(BBApiRequest& request) &&; +}; + +/** + * @struct ClientIvcAccumulate + * @brief Accumulate the previously loaded circuit into the IVC proof + */ +struct ClientIvcAccumulate { + static constexpr const char* NAME = "ClientIvcAccumulate"; + + /** + * @struct Response + * @brief Empty response indicating successful circuit accumulation + */ + struct Response { + static constexpr const char* NAME = "ClientIvcAccumulateResponse"; + // Empty response - success indicated by no exception + }; + + /** @brief Serialized witness data for the last loaded circuit */ + std::vector witness; + Response execute(BBApiRequest& request) &&; +}; + +/** + * @struct ClientIvcProve + * @brief Generate a proof for all accumulated circuits + */ +struct ClientIvcProve { + static constexpr const char* NAME = "ClientIvcProve"; + + /** + * @struct Response + * @brief Contains the generated IVC proof + */ + struct Response { + static constexpr const char* NAME = "ClientIvcProveResponse"; + + /** @brief Complete IVC proof for all accumulated circuits */ + ClientIVC::Proof proof; + }; + Response execute(BBApiRequest& request) &&; +}; + +/** + * @struct ClientIvcComputeStandaloneVk + * @brief Compute standalone verification key for a circuit + */ +struct ClientIvcComputeStandaloneVk { + static constexpr const char* NAME = "ClientIvcComputeStandaloneVk"; + + /** + * @struct Response + * @brief Contains the computed verification key in multiple formats + */ + struct Response { + static constexpr const char* NAME = "ClientIvcComputeStandaloneVkResponse"; + + /** @brief Serialized verification key in binary format */ + std::vector bytes; + /** @brief Verification key as array of field elements */ + std::vector fields; + }; + + /** @brief Circuit bytecode without precomputed VK */ + CircuitInputNoVK circuit; + Response execute(const BBApiRequest& request = {}) &&; +}; + +/** + * @struct ClientIvcComputeIvcVk + * @brief Compute IVC verification key for the complete proof + */ +struct ClientIvcComputeIvcVk { + static constexpr const char* NAME = "ClientIvcComputeIvcVk"; + + /** + * @struct Response + * @brief Contains the computed IVC verification key + */ + struct Response { + static constexpr const char* NAME = "ClientIvcComputeIvcVkResponse"; + + /** @brief Serialized IVC verification key in binary format */ + std::vector bytes; + }; + + /** @brief Final circuit bytecode for IVC VK computation */ + CircuitInputNoVK circuit; + Response execute(const BBApiRequest& request = {}) &&; +}; + +/** + * @struct ClientIvcCheckPrecomputedVk + * @brief Verify that a precomputed verification key matches the circuit + */ +struct ClientIvcCheckPrecomputedVk { + static constexpr const char* NAME = "ClientIvcCheckPrecomputedVk"; + + /** + * @struct Response + * @brief Contains the validation result + */ + struct Response { + static constexpr const char* NAME = "ClientIvcCheckPrecomputedVkResponse"; + + /** @brief True if the precomputed VK matches the circuit */ + bool valid; + }; + + /** @brief Circuit with its precomputed verification key */ + CircuitInput circuit; + /** @brief Human-readable name for logging and error messages */ + std::string function_name; + Response execute(const BBApiRequest& request = {}) &&; +}; + +} // namespace bb::bbapi diff --git a/barretenberg/cpp/src/barretenberg/api/bbapi_execute.hpp b/barretenberg/cpp/src/barretenberg/api/bbapi_execute.hpp new file mode 100644 index 000000000000..1e31053ceaeb --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/api/bbapi_execute.hpp @@ -0,0 +1,73 @@ +#pragma once + +#include "barretenberg/api/bbapi_client_ivc.hpp" +#include "barretenberg/api/bbapi_shared.hpp" +#include "barretenberg/api/bbapi_ultra_honk.hpp" +#include "barretenberg/common/throw_or_abort.hpp" +#include + +namespace bb::bbapi { + +using Command = NamedUnion; + +using CommandResponse = NamedUnion; + +/** + * @brief Executes a command by visiting a variant of all possible commands. + * + * @param command The command to execute, consumed by this function. + * @param request The circuit registry (acting as the request context). + * @return A variant of all possible command responses. + */ +inline CommandResponse execute(BBApiRequest& request, Command&& command) +{ + return std::move(command).visit([&request](auto&& cmd) -> CommandResponse { + using CmdType = std::decay_t; + return std::forward(cmd).execute(request); + }); +} + +// Can only be called from the execution thread (the same as the main thread, except in threaded WASM). +inline std::vector execute_request(BBApiRequest&& request, std::vector&& commands) +{ + std::vector responses; + responses.reserve(commands.size()); + for (Command& command : commands) { + responses.push_back(execute(request, std::move(command))); + } + return responses; +} + +} // namespace bb::bbapi diff --git a/barretenberg/cpp/src/barretenberg/api/bbapi_shared.hpp b/barretenberg/cpp/src/barretenberg/api/bbapi_shared.hpp new file mode 100644 index 000000000000..a2158c5f5fdf --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/api/bbapi_shared.hpp @@ -0,0 +1,131 @@ +#pragma once +/** + * @file bbapi_shared.hpp + * @brief Shared type definitions for the Barretenberg RPC API. + * + * This file contains common data structures used across multiple bbapi modules, + * including circuit input types and proof system settings. + */ +#include "barretenberg/client_ivc/client_ivc.hpp" +#include "barretenberg/dsl/acir_format/acir_format.hpp" +#include "barretenberg/honk/execution_trace/mega_execution_trace.hpp" +#include +#include +#include + +namespace bb::bbapi { + +/** + * @struct CircuitInputNoVK + * @brief A circuit to be used in either ultrahonk or chonk (ClientIVC+honk) verification key derivation. + */ +struct CircuitInputNoVK { + /** + * @brief Human-readable name for the circuit + * + * This name is not used for processing but serves as a debugging aid and + * provides context for circuit identification in logs and diagnostics. + */ + std::string name; + + /** + * @brief Serialized bytecode representation of the circuit + * + * Contains the ACIR program in serialized form. The format (bincode or msgpack) + * is determined by examining the first byte of the bytecode. + */ + std::vector bytecode; +}; + +/** + * @struct CircuitInput + * @brief A circuit to be used in either ultrahonk or ClientIVC-honk proving. + */ +struct CircuitInput { + /** + * @brief Human-readable name for the circuit + * + * This name is not used for processing but serves as a debugging aid and + * provides context for circuit identification in logs and diagnostics. + */ + std::string name; + + /** + * @brief Serialized bytecode representation of the circuit + * + * Contains the ACIR program in serialized form. The format (bincode or msgpack) + * is determined by examining the first byte of the bytecode. + */ + std::vector bytecode; + + /** + * @brief Verification key of the circuit. This could be derived, but it is more efficient to have it fixed ahead of + * time. As well, this guards against unexpected changes in the verification key. + */ + std::vector verification_key; +}; + +struct ProofSystemSettings { + /** + * @brief Optional flag to indicate if the proof should be generated with IPA accumulation (i.e. for rollup + * circuits). + */ + bool ipa_accumulation = false; + + /** + * @brief The oracle hash type to be used for the proof. + * + * This is used to determine the hash function used in the proof generation. + * Valid values are "poseidon2", "keccak", and "starknet". + */ + std::string oracle_hash_type = "poseidon2"; + + /** + * @brief Flag to disable blinding of the proof. + * Useful for cases that don't require privacy, such as when all inputs are public or zk-SNARK proofs themselves. + */ + bool disable_zk = false; + + /** + * @brief Honk recursion setting. + * 0 = no recursion, 1 = UltraHonk recursion, 2 = UltraRollupHonk recursion. + * Controls whether pairing point accumulators and IPA claims are added to public inputs. + */ + uint32_t honk_recursion = 0; + + /** + * @brief Flag to indicate if this circuit will be recursively verified. + */ + bool recursive = false; +}; + +/** + * @brief Convert oracle hash type string to enum for internal use + */ +enum class OracleHashType { POSEIDON2, KECCAK, STARKNET }; + +inline OracleHashType parse_oracle_hash_type(const std::string& type) +{ + if (type == "keccak") { + return OracleHashType::KECCAK; + } + if (type == "starknet") { + return OracleHashType::STARKNET; + } + return OracleHashType::POSEIDON2; // default +} + +struct BBApiRequest { + TraceSettings trace_settings{ AZTEC_TRACE_STRUCTURE }; + // Current depth of the IVC stack for this request + uint32_t ivc_stack_depth = 0; + std::shared_ptr ivc_in_progress; + // Name of the last loaded circuit + std::string loaded_circuit_name; + // Store the parsed constraint system to get ahead of parsing before accumulate + std::optional loaded_circuit_constraints; + // Store the verification key passed with the circuit + std::vector loaded_circuit_vk; +}; + +} // namespace bb::bbapi diff --git a/barretenberg/cpp/src/barretenberg/api/bbapi_ultra_honk.cpp b/barretenberg/cpp/src/barretenberg/api/bbapi_ultra_honk.cpp new file mode 100644 index 000000000000..280c26f2dbd0 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/api/bbapi_ultra_honk.cpp @@ -0,0 +1,69 @@ +#include "barretenberg/api/bbapi_ultra_honk.hpp" +#include "barretenberg/api/bbapi_shared.hpp" +#include "barretenberg/circuit_checker/circuit_checker.hpp" +#include "barretenberg/common/serialize.hpp" +#include "barretenberg/common/throw_or_abort.hpp" +#include "barretenberg/dsl/acir_format/acir_format.hpp" +#include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp" +#include "barretenberg/dsl/acir_format/serde/witness_stack.hpp" +#include "barretenberg/flavor/mega_flavor.hpp" +#include "barretenberg/honk/types/aggregation_object_type.hpp" +#include "barretenberg/ultra_honk/decider_proving_key.hpp" +#include "barretenberg/ultra_honk/ultra_prover.hpp" +#include "barretenberg/ultra_honk/ultra_verifier.hpp" +#include +#include + +namespace bb::bbapi { + +CircuitProve::Response CircuitProve::execute(BB_UNUSED const BBApiRequest& request) && +{ + throw_or_abort("not implemented yet!"); +} + +CircuitComputeVk::Response CircuitComputeVk::execute(BB_UNUSED const BBApiRequest& request) && +{ + throw_or_abort("not implemented yet!"); +} + +CircuitInfo::Response CircuitInfo::execute(BB_UNUSED const BBApiRequest& request) && +{ + throw_or_abort("not implemented yet!"); +} + +CircuitCheck::Response CircuitCheck::execute(BB_UNUSED const BBApiRequest& request) && +{ + throw_or_abort("not implemented yet!"); +} + +CircuitVerify::Response CircuitVerify::execute(BB_UNUSED const BBApiRequest& request) && +{ + throw_or_abort("not implemented yet!"); +} + +ProofAsFields::Response ProofAsFields::execute(BB_UNUSED const BBApiRequest& request) && +{ + throw_or_abort("not implemented yet!"); +} + +VkAsFields::Response VkAsFields::execute(BB_UNUSED const BBApiRequest& request) && +{ + throw_or_abort("not implemented yet!"); +} + +CircuitWriteSolidityVerifier::Response CircuitWriteSolidityVerifier::execute(BB_UNUSED const BBApiRequest& request) && +{ + throw_or_abort("not implemented yet"); +} + +CircuitProveAndVerify::Response CircuitProveAndVerify::execute(BB_UNUSED const BBApiRequest& request) && +{ + throw_or_abort("not implemented yet!"); +} + +CircuitBenchmark::Response CircuitBenchmark::execute(BB_UNUSED const BBApiRequest& request) && +{ + throw_or_abort("not implemented yet!"); +} + +} // namespace bb::bbapi diff --git a/barretenberg/cpp/src/barretenberg/api/bbapi_ultra_honk.hpp b/barretenberg/cpp/src/barretenberg/api/bbapi_ultra_honk.hpp new file mode 100644 index 000000000000..c218ba6746dc --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/api/bbapi_ultra_honk.hpp @@ -0,0 +1,224 @@ +#pragma once +/** + * @file bbapi_ultra_honk.hpp + * @brief UltraHonk-specific command definitions for the Barretenberg RPC API. + * + * This file contains command structures for UltraHonk proof system operations + * including circuit proving, verification, VK computation, and utility functions. + */ +#include "barretenberg/api/bbapi_shared.hpp" +#include "barretenberg/common/named_union.hpp" +#include "barretenberg/honk/proof_system/types/proof.hpp" +#include +#include + +namespace bb::bbapi { + +// CircuitInput, CircuitInputNoVK, and ProofSystemSettings are defined in bbapi_shared.hpp + +/** + * @struct CircuitProve + * @brief Represents a request to generate a proof. + * Currently, UltraHonk is the only proving system supported by BB (after plonk was deprecated and removed). + * This is used for one-shot proving, not our "IVC" scheme, ClientIVC-honk. For that, use the ClientIVC* commands. + * + * This structure is used to encapsulate all necessary parameters for generating a proof + * for a specific circuit, including the circuit bytecode, verification key, witness data, and options for the proving + * process. + */ +struct CircuitProve { + static constexpr const char* NAME = "CircuitProve"; + + /** + * @brief Contains proof and public inputs. + * Both are given as vectors of fields. To be used for verification. + * Example uses of this Response would be verification in native BB, WASM BB, solidity or recursively through Noir. + */ + struct Response { + static constexpr const char* NAME = "CircuitProveResponse"; + + PublicInputsVector public_inputs; + HonkProof proof; + }; + + CircuitInput circuit; + std::vector witness; + ProofSystemSettings settings; + Response execute(const BBApiRequest& request = {}) &&; +}; + +struct CircuitComputeVk { + static constexpr const char* NAME = "CircuitComputeVk"; + + struct Response { + static constexpr const char* NAME = "CircuitComputeVkResponse"; + + std::vector bytes; // Serialized verification key + }; + + CircuitInputNoVK circuit; + ProofSystemSettings settings; + Response execute(const BBApiRequest& request = {}) &&; +}; + +/** + * @struct CircuitInfo + * @brief Consolidated command for retrieving circuit information. + * Combines gate count, circuit size, and other metadata into a single command. + */ +struct CircuitInfo { + static constexpr const char* NAME = "CircuitInfo"; + + struct Response { + static constexpr const char* NAME = "CircuitInfoResponse"; + + uint32_t total_gates; + uint32_t subgroup_size; + std::map gates_per_opcode; // Optional: gate counts per opcode + }; + + CircuitInput circuit; + bool include_gates_per_opcode = false; + ProofSystemSettings settings; + Response execute(const BBApiRequest& request = {}) &&; +}; + +/** + * @struct CircuitCheck + * @brief Verify that a witness satisfies a circuit's constraints. + * For debugging and validation purposes. + */ +struct CircuitCheck { + static constexpr const char* NAME = "CircuitCheck"; + + struct Response { + static constexpr const char* NAME = "CircuitCheckResponse"; + + bool satisfied; + }; + + CircuitInput circuit; + std::vector witness; + ProofSystemSettings settings; + Response execute(const BBApiRequest& request = {}) &&; +}; + +/** + * @struct CircuitVerify + * @brief Verify a proof against a verification key and public inputs. + */ +struct CircuitVerify { + static constexpr const char* NAME = "CircuitVerify"; + + struct Response { + static constexpr const char* NAME = "CircuitVerifyResponse"; + + bool verified; + }; + + std::vector verification_key; + PublicInputsVector public_inputs; + HonkProof proof; + ProofSystemSettings settings; + Response execute(const BBApiRequest& request = {}) &&; +}; + +/** + * @struct ProofAsFields + * @brief Convert a proof to field elements representation. + */ +struct ProofAsFields { + static constexpr const char* NAME = "ProofAsFields"; + + struct Response { + static constexpr const char* NAME = "ProofAsFieldsResponse"; + + std::vector fields; + }; + + HonkProof proof; + Response execute(const BBApiRequest& request = {}) &&; +}; + +/** + * @struct VkAsFields + * @brief Convert a verification key to field elements representation. + */ +struct VkAsFields { + static constexpr const char* NAME = "VkAsFields"; + + struct Response { + static constexpr const char* NAME = "VkAsFieldsResponse"; + + std::vector fields; + }; + + std::vector verification_key; + bool is_mega_honk = false; + Response execute(const BBApiRequest& request = {}) &&; +}; + +/** + * @brief Command to generate Solidity verifier contract + */ +struct CircuitWriteSolidityVerifier { + static constexpr const char* NAME = "CircuitWriteSolidityVerifier"; + + struct Response { + static constexpr const char* NAME = "CircuitWriteSolidityVerifierResponse"; + + std::string solidity_code; + }; + + std::vector verification_key; + ProofSystemSettings settings; + Response execute(const BBApiRequest& request = {}) &&; +}; + +/** + * @brief Command to prove and verify in one step + */ +struct CircuitProveAndVerify { + static constexpr const char* NAME = "CircuitProveAndVerify"; + + struct Response { + static constexpr const char* NAME = "CircuitProveAndVerifyResponse"; + + bool verified; + HonkProof proof; + PublicInputsVector public_inputs; + }; + + CircuitInput circuit; + std::vector witness; + ProofSystemSettings settings; + Response execute(const BBApiRequest& request = {}) &&; +}; + +/** + * @brief Command to benchmark circuit operations + */ +struct CircuitBenchmark { + static constexpr const char* NAME = "CircuitBenchmark"; + + struct Response { + static constexpr const char* NAME = "CircuitBenchmarkResponse"; + + double witness_generation_time_ms; + double proving_time_ms; + double verification_time_ms; + uint64_t peak_memory_bytes; + }; + + CircuitInput circuit; + std::vector witness; + ProofSystemSettings settings; + uint32_t num_iterations = 1; + bool benchmark_witness_generation = true; + bool benchmark_proving = true; + Response execute(const BBApiRequest& request = {}) &&; +}; + +// OracleHashType enum and parse_oracle_hash_type are defined in bbapi_shared.hpp + +} // namespace bb::bbapi diff --git a/barretenberg/cpp/src/barretenberg/api/bbrpc_commands.hpp b/barretenberg/cpp/src/barretenberg/api/bbrpc_commands.hpp deleted file mode 100644 index 1963a7961050..000000000000 --- a/barretenberg/cpp/src/barretenberg/api/bbrpc_commands.hpp +++ /dev/null @@ -1,515 +0,0 @@ -#pragma once -/** - * @file bbrpc.hpp - * @brief Barretenberg RPC provides a stateful API for all core barretenberg proving functions. - * Not included: - * - Solidity verifier generation - * - Raw cryptography functions exposed by WASM BB - */ -#include "barretenberg/client_ivc/client_ivc.hpp" -#include "barretenberg/common/named_union.hpp" -#include "barretenberg/honk/proof_system/types/proof.hpp" -#include - -namespace bb::bbrpc { - -/** - * @struct CircuitInputNoVK - * @brief A circuit to be used in either ultrahonk or chonk (ClientIVC+honk) verification key derivation. - */ -struct CircuitInputNoVK { - /** - * @brief Human-readable name for the circuit - * - * This name is not used for processing but serves as a debugging aid and - * provides context for circuit identification in logs and diagnostics. - */ - std::string name; - - /** - * @brief Serialized bytecode representation of the circuit - * - * Contains the ACIR program in serialized form. The format (bincode or msgpack) - * is determined by examining the first byte of the bytecode. - */ - std::vector bytecode; -}; - -/** - * @struct CircuitInput - * @brief A circuit to be used in either ultrahonk or ClientIVC-honk proving. - */ -struct CircuitInput { - /** - * @brief Human-readable name for the circuit - * - * This name is not used for processing but serves as a debugging aid and - * provides context for circuit identification in logs and diagnostics. - */ - std::string name; - - /** - * @brief Serialized bytecode representation of the circuit - * - * Contains the ACIR program in serialized form. The format (bincode or msgpack) - * is determined by examining the first byte of the bytecode. - */ - std::vector bytecode; - - /** - * @brief Verification key of the circuit. This could be derived, but it is more efficient to have it fixed ahead of - * time. As well, this guards against unexpected changes in the verification key. - */ - std::vector verification_key; -}; - -struct ProofSystemSettings { - /** - * @brief Optional flag to indicate if the proof should be generated with IPA accumulation (i.e. for rollup - * circuits). - */ - bool ipa_accumulation = false; - - /** - * @brief The oracle hash type to be used for the proof. - * - * This is used to determine the hash function used in the proof generation. - * Valid values are "poseidon2", "keccak", and "starknet". - */ - std::string oracle_hash_type = "poseidon2"; - - /** - * @brief Flag to disable blinding of the proof. - * Useful for cases that don't require privacy, such as when all inputs are public or zk-SNARK proofs themselves. - */ - bool disable_zk = false; - - /** - * @brief Honk recursion setting. - * 0 = no recursion, 1 = UltraHonk recursion, 2 = UltraRollupHonk recursion. - * Controls whether pairing point accumulators and IPA claims are added to public inputs. - */ - uint32_t honk_recursion = 0; - - /** - * @brief Flag to indicate if this circuit will be recursively verified. - */ - bool recursive = false; -}; - -/** - * @struct CircuitProve - * @brief Represents a request to generate a proof. - * Currently, UltraHonk is the only proving system supported by BB (after plonk was deprecated and removed). - * This is used for one-shot proving, not our "IVC" scheme, ClientIVC-honk. For that, use the ClientIVC* commands. - * - * This structure is used to encapsulate all necessary parameters for generating a proof - * for a specific circuit, including the circuit bytecode, verification key, witness data, and options for the proving - * process. - */ -struct CircuitProve { - static constexpr const char* NAME = "CircuitProve"; - - /** - * @brief Contains proof and public inputs. - * Both are given as vectors of fields. To be used for verification. - * Example uses of this Response would be verification in native BB, WASM BB, solidity or recursively through Noir. - */ - struct Response { - static constexpr const char* NAME = "CircuitProveResponse"; - - PublicInputsVector public_inputs; - HonkProof proof; - // Empty if successful. - std::string error_message; - }; - - CircuitInput circuit; - std::vector witness; - ProofSystemSettings settings; -}; - -struct CircuitComputeVk { - static constexpr const char* NAME = "CircuitComputeVk"; - - struct Response { - static constexpr const char* NAME = "CircuitComputeVkResponse"; - - /** - * @brief Serialized verification key. - */ - std::vector verification_key; - // Empty if successful. - std::string error_message; - }; - - CircuitInputNoVK circuit; - ProofSystemSettings settings; -}; - -/** Compute verification key, Treat the previously loaded circuit as either a standalone circuit - * or a common final circuit used to verify all of IVC. */ -struct CircuitComputeIvcVk { - static constexpr const char* NAME = "CircuitComputeIvcVk"; - - struct Response { - static constexpr const char* NAME = "CircuitComputeIvcVkResponse"; - - /** - * @brief Serialized verification key. - */ - std::vector verification_key; - // Empty if successful. - std::string error_message; - }; - bool standalone; -}; - -/** Compute verification key, Treat the previously loaded circuit as either a standalone circuit - * or a common final circuit used to verify all of IVC. */ -struct ClientIvcComputeVk { - static constexpr const char* NAME = "ClientIvcComputeVk"; - - struct Response { - static constexpr const char* NAME = "ClientIvcComputeVkResponse"; - - /** - * @brief Serialized verification key. - */ - std::vector verification_key; - // Empty if successful. - std::string error_message; - }; - - CircuitInputNoVK circuit; - bool standalone; -}; - -/** - * @brief - * Note, only one IVC request can be made at a time for each batch_request. - */ -struct ClientIvcStart { - static constexpr const char* NAME = "ClientIvcStart"; - - struct Response { - static constexpr const char* NAME = "ClientIvcStartResponse"; - - // Empty if successful. - std::string error_message; - }; -}; - -struct ClientIvcLoad { - static constexpr const char* NAME = "ClientIvcLoad"; - - struct Response { - static constexpr const char* NAME = "ClientIvcLoadResponse"; - - // Empty if successful. - std::string error_message; - }; - - CircuitInput circuit; -}; - -struct ClientIvcAccumulate { - static constexpr const char* NAME = "ClientIvcAccumulate"; - - struct Response { - static constexpr const char* NAME = "ClientIvcAccumulateResponse"; - - // Empty if successful. - std::string error_message; - }; - - // Serialized witness for the last loaded circuit. - std::vector witness; -}; - -struct ClientIvcProve { - static constexpr const char* NAME = "ClientIvcProve"; - - struct Response { - static constexpr const char* NAME = "ClientIvcProveResponse"; - - ClientIVC::Proof proof; - // Empty if successful. - std::string error_message; - }; -}; - -/** - * @struct CircuitInfo - * @brief Consolidated command for retrieving circuit information. - * Combines gate count, circuit size, and other metadata into a single command. - */ -struct CircuitInfo { - static constexpr const char* NAME = "CircuitInfo"; - - struct Response { - static constexpr const char* NAME = "CircuitInfoResponse"; - - uint32_t total_gates; - uint32_t subgroup_size; - // Optional: gate counts per opcode - std::map gates_per_opcode; - // Empty if successful. - std::string error_message; - }; - - CircuitInput circuit; - bool include_gates_per_opcode = false; - ProofSystemSettings settings; -}; - -/** - * @struct CircuitCheck - * @brief Verify that a witness satisfies a circuit's constraints. - * For debugging and validation purposes. - */ -struct CircuitCheck { - static constexpr const char* NAME = "CircuitCheck"; - - struct Response { - static constexpr const char* NAME = "CircuitCheckResponse"; - - bool satisfied; - // Empty if successful, contains constraint failure details if not satisfied. - std::string error_message; - }; - - CircuitInput circuit; - std::vector witness; - ProofSystemSettings settings; -}; - -/** - * @struct CircuitVerify - * @brief Verify a proof against a verification key and public inputs. - */ -struct CircuitVerify { - static constexpr const char* NAME = "CircuitVerify"; - - struct Response { - static constexpr const char* NAME = "CircuitVerifyResponse"; - - bool verified; - // Empty if successful. - std::string error_message; - }; - - std::vector verification_key; - PublicInputsVector public_inputs; - HonkProof proof; - ProofSystemSettings settings; -}; - -/** - * @struct ProofAsFields - * @brief Convert a proof to field elements representation. - */ -struct ProofAsFields { - static constexpr const char* NAME = "ProofAsFields"; - - struct Response { - static constexpr const char* NAME = "ProofAsFieldsResponse"; - - std::vector fields; - // Empty if successful. - std::string error_message; - }; - - HonkProof proof; -}; - -/** - * @struct VkAsFields - * @brief Convert a verification key to field elements representation. - */ -struct VkAsFields { - static constexpr const char* NAME = "VkAsFields"; - - struct Response { - static constexpr const char* NAME = "VkAsFieldsResponse"; - - std::vector fields; - // Empty if successful. - std::string error_message; - }; - - std::vector verification_key; - bool is_mega_honk = false; -}; - -/** - * @brief Command to generate Solidity verifier contract - */ -struct CircuitWriteSolidityVerifier { - static constexpr const char* NAME = "CircuitWriteSolidityVerifier"; - - struct Response { - static constexpr const char* NAME = "CircuitWriteSolidityVerifierResponse"; - - std::string solidity_code; - std::string error_message; - }; - - std::vector verification_key; - ProofSystemSettings settings; -}; - -/** - * @brief Command to prove and verify in one step - */ -struct CircuitProveAndVerify { - static constexpr const char* NAME = "CircuitProveAndVerify"; - - struct Response { - static constexpr const char* NAME = "CircuitProveAndVerifyResponse"; - - bool verified; - std::vector proof; // The generated proof - std::vector public_inputs; // Extracted public inputs - std::string error_message; - }; - - CircuitInput circuit; - std::vector witness; - ProofSystemSettings settings; -}; - -/** - * @brief Command to write circuit bytecode in various formats - */ -struct CircuitWriteBytecode { - static constexpr const char* NAME = "CircuitWriteBytecode"; - - struct Response { - static constexpr const char* NAME = "CircuitWriteBytecodeResponse"; - - std::vector bytecode; - std::string formatted_output; // For hex/base64 - std::string error_message; - }; - - CircuitInput circuit; - std::string format = "binary"; // binary, hex, base64 -}; - -/** - * @brief Command to validate circuit structure - */ -struct CircuitValidate { - static constexpr const char* NAME = "CircuitValidate"; - - struct Response { - static constexpr const char* NAME = "CircuitValidateResponse"; - - bool is_valid; - std::vector validation_errors; - std::string error_message; - }; - - CircuitInput circuit; - ProofSystemSettings settings; - bool check_recursive_structure = false; -}; - -/** - * @brief Command to benchmark circuit operations - */ -struct CircuitBenchmark { - static constexpr const char* NAME = "CircuitBenchmark"; - - struct Response { - static constexpr const char* NAME = "CircuitBenchmarkResponse"; - - double witness_generation_time_ms; - double proving_time_ms; - double verification_time_ms; - uint64_t peak_memory_bytes; - std::string error_message; - }; - - CircuitInput circuit; - std::vector witness; - ProofSystemSettings settings; - uint32_t num_iterations = 1; - bool benchmark_witness_generation = true; - bool benchmark_proving = true; -}; - -/** - * @brief Command to check if a precomputed VK matches the circuit - */ -struct ClientIvcCheckPrecomputedVk { - static constexpr const char* NAME = "ClientIvcCheckPrecomputedVk"; - - struct Response { - static constexpr const char* NAME = "ClientIvcCheckPrecomputedVkResponse"; - - bool valid; - std::string error_message; - }; - - // Circuit with its precomputed VK - CircuitInput circuit; - std::string function_name; -}; - -using Command = NamedUnion; - -using CommandResponse = NamedUnion; - -/** - * @brief Convert oracle hash type string to enum for internal use - */ -enum class OracleHashType { POSEIDON2, KECCAK, STARKNET }; - -inline OracleHashType parse_oracle_hash_type(const std::string& type) -{ - if (type == "keccak") { - return OracleHashType::KECCAK; - } - if (type == "starknet") { - return OracleHashType::STARKNET; - } - return OracleHashType::POSEIDON2; // default -} - -} // namespace bb::bbrpc diff --git a/barretenberg/cpp/src/barretenberg/api/bbrpc_execute.hpp b/barretenberg/cpp/src/barretenberg/api/bbrpc_execute.hpp deleted file mode 100644 index c9ef03a7afed..000000000000 --- a/barretenberg/cpp/src/barretenberg/api/bbrpc_execute.hpp +++ /dev/null @@ -1,219 +0,0 @@ -#pragma once - -#include "barretenberg/api/bbrpc_commands.hpp" -#include "barretenberg/api/write_prover_output.hpp" -#include "barretenberg/circuit_checker/circuit_checker.hpp" -#include "barretenberg/client_ivc/client_ivc.hpp" -#include "barretenberg/client_ivc/mock_circuit_producer.hpp" -#include "barretenberg/common/compiler_hints.hpp" -#include "barretenberg/common/log.hpp" -#include "barretenberg/common/throw_or_abort.hpp" -#include "barretenberg/dsl/acir_format/acir_format.hpp" -#include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp" -#include "barretenberg/dsl/acir_format/serde/witness_stack.hpp" -#include "barretenberg/honk/execution_trace/mega_execution_trace.hpp" -#include "barretenberg/serialize/msgpack_check_eq.hpp" -#include "barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp" -#include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp" -#include -#include - -namespace bb::bbrpc { - -struct BBRpcRequest { - TraceSettings trace_settings{ AZTEC_TRACE_STRUCTURE }; - // Current depth of the IVC stack for this request - uint32_t ivc_stack_depth = 0; - std::shared_ptr ivc_in_progress; - // Name of the last loaded circuit - std::string last_circuit_name; - // Store the parsed constraint system to get ahead of parsing before accumulate - std::optional last_circuit_constraints; - // Store the verification key passed with the circuit - std::vector last_circuit_vk; -}; - -inline const std::string& get_error_message(const CommandResponse& response) -{ - return response.visit([](const auto& resp) -> const std::string& { return resp.error_message; }); -} - -inline CircuitProve::Response execute(BB_UNUSED BBRpcRequest& request, CircuitProve&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -inline CircuitComputeVk::Response execute(BB_UNUSED BBRpcRequest& request, CircuitComputeVk&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -inline CircuitVerify::Response execute(BB_UNUSED BBRpcRequest& request, CircuitVerify&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -inline CircuitInfo::Response execute(BB_UNUSED BBRpcRequest& request, CircuitInfo&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -inline CircuitCheck::Response execute(BB_UNUSED BBRpcRequest& request, CircuitCheck&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -inline ProofAsFields::Response execute(BB_UNUSED BBRpcRequest& request, ProofAsFields&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -inline VkAsFields::Response execute(BB_UNUSED BBRpcRequest& request, VkAsFields&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -inline ClientIvcStart::Response execute(BBRpcRequest& request, BB_UNUSED ClientIvcStart&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -inline ClientIvcLoad::Response execute(BBRpcRequest& request, ClientIvcLoad&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -inline ClientIvcAccumulate::Response execute(BBRpcRequest& request, ClientIvcAccumulate&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -inline ClientIvcProve::Response execute(BBRpcRequest& request, ClientIvcProve&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -inline std::shared_ptr get_acir_program_decider_proving_key( - const BBRpcRequest& request, acir_format::AcirProgram& program) -{ - ClientIVC::ClientCircuit builder = acir_format::create_circuit(program); - - // Construct the verification key via the prover-constructed proving key with the proper trace settings - return std::make_shared(builder, request.trace_settings); -} - -inline ClientIVC::VerificationKey compute_vk_for_ivc(const BBRpcRequest& request, - size_t num_public_inputs_in_final_circuit) -{ - ClientIVC ivc{ /*num_circuits=*/2, request.trace_settings }; - ClientIVCMockCircuitProducer circuit_producer; - - // Initialize the IVC with an arbitrary circuit - // We segfault if we only call accumulate once - static constexpr size_t SMALL_ARBITRARY_LOG_CIRCUIT_SIZE{ 5 }; - MegaCircuitBuilder circuit_0 = circuit_producer.create_next_circuit(ivc, SMALL_ARBITRARY_LOG_CIRCUIT_SIZE); - ivc.accumulate(circuit_0); - - // Create another circuit and accumulate - MegaCircuitBuilder circuit_1 = - circuit_producer.create_next_circuit(ivc, SMALL_ARBITRARY_LOG_CIRCUIT_SIZE, num_public_inputs_in_final_circuit); - ivc.accumulate(circuit_1); - - // Construct the hiding circuit and its VK (stored internally in the IVC) - ivc.construct_hiding_circuit_key(); - - return ivc.get_vk(); -} - -inline ClientIvcComputeVk::Response execute(BBRpcRequest& request, ClientIvcComputeVk&& command) -{ - info("ClientIvcComputeVk - deriving VK for circuit '", command.circuit.name, "', standalone: ", command.standalone); - - // Parse the circuit - auto constraint_system = acir_format::circuit_buf_to_acir_format(std::move(command.circuit.bytecode)); - - // Create verification key based on whether it's standalone or not - std::vector vk_data; - if (command.standalone) { - // For standalone, we just need the circuit's verification key (not the full IVC VK) - acir_format::AcirProgram program{ constraint_system, /*witness=*/{} }; - std::shared_ptr proving_key = - get_acir_program_decider_proving_key(request, program); - auto verification_key = std::make_shared(proving_key->get_precomputed()); - vk_data = to_buffer(*verification_key); - info("ClientIvcComputeVk - standalone VK derived, size: ", vk_data.size(), " bytes"); - } else { - vk_data = to_buffer(compute_vk_for_ivc(request, constraint_system.public_inputs.size())); - info("ClientIvcComputeVk - full IVC VK derived, size: ", vk_data.size(), " bytes"); - } - - return ClientIvcComputeVk::Response{ .verification_key = vk_data, .error_message = "" }; -} - -inline ClientIvcCheckPrecomputedVk::Response execute(BBRpcRequest& request, ClientIvcCheckPrecomputedVk&& command) -{ - (void)request; - (void)command; - throw_or_abort("code in progress! should not be called"); -} - -/** - * @brief Executes a command by visiting a variant of all possible commands. - * - * @param command The command to execute, consumed by this function. - * @param request The circuit registry (acting as the request context). - * @return A variant of all possible command responses. - */ -inline CommandResponse execute(BBRpcRequest& request, Command&& command) -{ - return std::move(command).visit( - [&request](auto&& cmd) -> CommandResponse { return execute(request, std::forward(cmd)); }); -} - -template typename T::Response execute_or_throw(BBRpcRequest& request, T&& command) -{ - auto response = execute(request, std::forward(command)); - if (!response.error_message.empty()) { - throw_or_abort(response.error_message); - } - return response; -} - -// Can only be called from the execution thread (the same as the main thread, except in threaded WASM). -inline std::vector execute_request(BBRpcRequest&& request, std::vector&& commands) -{ - std::vector responses; - responses.reserve(commands.size()); - for (Command& command : commands) { - responses.push_back(execute(request, std::move(command))); - if (!get_error_message(responses.back()).empty()) { - // If there was an error, we stop processing further commands. - break; - } - } - return responses; -} - -} // namespace bb::bbrpc diff --git a/barretenberg/cpp/src/barretenberg/api/write_prover_output.hpp b/barretenberg/cpp/src/barretenberg/api/write_prover_output.hpp index cf9370b1a52d..436a9e50358a 100644 --- a/barretenberg/cpp/src/barretenberg/api/write_prover_output.hpp +++ b/barretenberg/cpp/src/barretenberg/api/write_prover_output.hpp @@ -10,6 +10,20 @@ namespace bb { +inline std::string field_elements_to_json(const std::vector& fields) +{ + std::stringstream ss; + ss << "["; + for (size_t i = 0; i < fields.size(); ++i) { + ss << '"' << fields[i] << '"'; + if (i < fields.size() - 1) { + ss << ","; + } + } + ss << "]"; + return ss.str(); +} + template struct PubInputsProofAndKey { PublicInputsVector public_inputs; HonkProof proof; diff --git a/barretenberg/cpp/src/barretenberg/bb/cli.cpp b/barretenberg/cpp/src/barretenberg/bb/cli.cpp index ec0e4708fef4..ac7d74bf2617 100644 --- a/barretenberg/cpp/src/barretenberg/bb/cli.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/cli.cpp @@ -578,7 +578,7 @@ int parse_and_run_cli_command(int argc, char* argv[]) // Immediately after parsing, we can init the global CRS factory. Note this does not yet read or download any // points; that is done on-demand. srs::init_net_crs_factory(flags.crs_path); - if (prove->parsed() || write_vk->parsed()) { + if ((prove->parsed() || write_vk->parsed()) && output_path != "-") { // If writing to an output folder, make sure it exists. std::filesystem::create_directories(output_path); } diff --git a/barretenberg/cpp/src/barretenberg/polynomials/backing_memory.hpp b/barretenberg/cpp/src/barretenberg/polynomials/backing_memory.hpp index 679ee92cde59..57663c48db74 100644 --- a/barretenberg/cpp/src/barretenberg/polynomials/backing_memory.hpp +++ b/barretenberg/cpp/src/barretenberg/polynomials/backing_memory.hpp @@ -8,6 +8,7 @@ #include "barretenberg/common/slab_allocator.hpp" #include "barretenberg/common/throw_or_abort.hpp" +#include "unistd.h" #include #include #include diff --git a/bootstrap.sh b/bootstrap.sh index 7a81f5798ce4..81f36ba7c813 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -240,8 +240,28 @@ function build { aztec-up/bootstrap.sh ) + local start_building=false for project in "${serial_projects[@]}"; do - $project/bootstrap.sh ${1:-} + # BOOTSTRAP_AFTER and BOOTSTRAP_TO are used to control the order of building. + # If BOOTSTRAP_AFTER is set, it should be one of our serial projects and we will only build projects after it. + # If BOOTSTRAP_TO is set, it should be one of our serial projects and we will only build projects up to it. We will skip parallel_cmds. + + # Start building after we've seen BOOTSTRAP_AFTER, skipping BOOTSTRAP_AFTER itself. + if [ "$project" == "${BOOTSTRAP_AFTER:-}" ]; then + start_building=true + continue + fi + + # Build the project if we should be building + if [[ -z "${BOOTSTRAP_AFTER:-}" || "$start_building" = true ]]; then + $project/bootstrap.sh ${1:-} + fi + + # Stop the build if we've reached BOOTSTRAP_TO + # We therefore don't run parallel commands if BOOTSTRAP_TO is set. + if [ "$project" = "${BOOTSTRAP_TO:-}" ]; then + return + fi done parallel --line-buffer --tag --halt now,fail=1 "denoise '{}'" ::: ${parallel_cmds[@]} diff --git a/ci3/aws_request_instance b/ci3/aws_request_instance index 0765c2548c40..c0f1fb98b7f6 100755 --- a/ci3/aws_request_instance +++ b/ci3/aws_request_instance @@ -47,6 +47,10 @@ else exit 1 fi +if [ -n "${AWS_INSTANCE:-}" ]; then + instances=("$AWS_INSTANCE") +fi + set +e IFS=',' read -ra cpu_list <<< "$cpus" for cpu in "${cpu_list[@]}"; do diff --git a/noir-projects/scripts/generate_vk_json.js b/noir-projects/scripts/generate_vk_json.js deleted file mode 100644 index 3fb3575ca42f..000000000000 --- a/noir-projects/scripts/generate_vk_json.js +++ /dev/null @@ -1,211 +0,0 @@ -const path = require("path"); -const fs = require("fs/promises"); -const child_process = require("child_process"); -const crypto = require("crypto"); - -const clientIvcPatterns = require("../client_ivc_circuits.json"); -const rollupHonkPatterns = require("../rollup_honk_circuits.json"); - -const CircuitType = { - ClientIVCCircuit: 0, - RollupHonkCircuit: 1, - HonkCircuit: 2, -}; - -const { - readVKFromS3, - writeVKToS3, - getBarretenbergHash, - generateArtifactHash, - BB_BIN_PATH, -} = require("./verification_keys"); - -function vkBinaryFileNameForArtifactName(outputFolder, artifactName) { - return path.join(outputFolder, `${artifactName}.vk`); -} - -function vkJsonFileNameForArtifactName(outputFolder, artifactName) { - return path.join(outputFolder, `${artifactName}.vk.json`); -} - -function vkDataFileNameForArtifactName(outputFolder, artifactName) { - return path.join(outputFolder, `${artifactName}.vk.data.json`); -} - -async function getBytecodeHash(artifactPath) { - const { bytecode } = JSON.parse(await fs.readFile(artifactPath)); - if (!bytecode) { - throw new Error("No bytecode found in artifact: " + artifactPath); - } - return crypto.createHash("md5").update(bytecode).digest("hex"); -} - -async function getArtifactHash(artifactPath, circuitType, isRecursive) { - const bytecodeHash = await getBytecodeHash(artifactPath); - const barretenbergHash = await getBarretenbergHash(); - return generateArtifactHash( - barretenbergHash, - bytecodeHash, - circuitType, - isRecursive - ); -} - -async function hasArtifactHashChanged(artifactHash, vkDataPath) { - try { - const { artifactHash: previousArtifactHash } = JSON.parse( - await fs.readFile(vkDataPath, "utf8") - ); - if (previousArtifactHash === artifactHash) { - return false; - } else { - console.log( - `Circuit ${artifactName} has changed, old hash ${previousArtifactHash}, new hash ${artifactHash}` - ); - } - } catch (ignored) { - console.log(`No on disk vk found in: ${vkDataPath}`); - } - return true; -} - -function typeOfCircuit(artifactName) { - if ( - clientIvcPatterns.some((pattern) => artifactName.match(new RegExp(pattern))) - ) { - return CircuitType.ClientIVCCircuit; - } else if ( - rollupHonkPatterns.some((pattern) => - artifactName.match(new RegExp(pattern)) - ) - ) { - return CircuitType.RollupHonkCircuit; - } else { - return CircuitType.HonkCircuit; - } -} - -async function processArtifact(artifactPath, artifactName, outputFolder) { - const circuitType = typeOfCircuit(artifactName); - const isRecursive = true; - - const artifactHash = await getArtifactHash( - artifactPath, - circuitType, - isRecursive - ); - - const vkDataPath = vkDataFileNameForArtifactName(outputFolder, artifactName); - - const hasChanged = await hasArtifactHashChanged(artifactHash, vkDataPath); - if (!hasChanged) { - console.log(`Reusing on disk vk: ${vkDataPath}`); - return; - } - - let vkData = await readVKFromS3(artifactName, artifactHash); - if (!vkData) { - vkData = await generateVKData( - artifactName, - outputFolder, - artifactPath, - artifactHash, - circuitType, - isRecursive - ); - await writeVKToS3(artifactName, artifactHash, JSON.stringify(vkData)); - } else { - console.log("Using VK from remote cache for", artifactName); - } - - await fs.writeFile(vkDataPath, JSON.stringify(vkData, null, 2)); -} - -async function generateVKData( - artifactName, - outputFolder, - artifactPath, - artifactHash, - circuitType, - isRecursive -) { - if (circuitType == CircuitType.ClientIVCCircuit) { - console.log("Generating new client ivc vk for", artifactName); - } else if (circuitType == CircuitType.RollupHonkCircuit) { - console.log("Generating new rollup honk vk for", artifactName); - } else { - console.log("Generating new honk vk for", artifactName); - } - - const binaryVkPath = vkBinaryFileNameForArtifactName( - outputFolder, - artifactName - ); - const jsonVkPath = vkJsonFileNameForArtifactName(outputFolder, artifactName); - - if (circuitType == CircuitType.ClientIVCCircuit) { - write_vk_flow = "write_vk_for_ivc"; - vk_as_fields_flow = "vk_as_fields_mega_honk"; - honk_recursion = 0; - } else if (circuitType == CircuitType.RollupHonkCircuit) { - write_vk_flow = "write_vk_ultra_rollup_honk"; - vk_as_fields_flow = "vk_as_fields_ultra_rollup_honk"; - honk_recursion = 2; - } else { - write_vk_flow = "write_vk --scheme ultra_honk"; - vk_as_fields_flow = "vk_as_fields_ultra_honk"; - honk_recursion = 1; - } - - const writeVkCommand = `${BB_BIN_PATH} ${write_vk_flow} -h ${honk_recursion} -b "${artifactPath}" -o "${binaryVkPath}"`; - - console.log("WRITE VK CMD: ", writeVkCommand); - - const vkAsFieldsCommand = `${BB_BIN_PATH} ${vk_as_fields_flow} -k "${binaryVkPath}" -o "${jsonVkPath}"`; - - console.log("VK AS FIELDS CMD: ", vkAsFieldsCommand); - - await new Promise((resolve, reject) => { - child_process.exec(`${writeVkCommand} && ${vkAsFieldsCommand}`, (err) => { - if (err) { - reject(err); - } else { - resolve(); - } - }); - }); - const binaryVk = await fs.readFile(binaryVkPath); - const jsonVk = JSON.parse(await fs.readFile(jsonVkPath, "utf8")); - await fs.unlink(jsonVkPath); - await fs.unlink(binaryVkPath); - - const vkData = { - keyAsBytes: binaryVk.toString("hex"), - keyAsFields: jsonVk, - artifactHash, - }; - console.log("Generated vk for", artifactName); - - return vkData; -} - -async function main() { - let [artifactPath, outputFolder] = process.argv.slice(2); - if (!artifactPath || !outputFolder) { - console.log( - "Usage: node generate_vk_json.js " - ); - return; - } - - await processArtifact( - artifactPath, - path.basename(artifactPath, ".json"), - outputFolder - ); -} - -main().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/scripts/auto_rebase_pr.sh b/scripts/auto_rebase_pr.sh index 27bdab852fbc..e95e8ab1b54d 100755 --- a/scripts/auto_rebase_pr.sh +++ b/scripts/auto_rebase_pr.sh @@ -1,263 +1,76 @@ #!/usr/bin/env bash set -euo pipefail -############################################################################### -# helpers -############################################################################### -function die { echo "❌ $*" >&2; exit 1; } -function is_merge { [[ $(git rev-list --parents -n1 "$1" | wc -w) -gt 2 ]]; } -function has_pr_suffix { [[ $(git show -s --format=%s "$1") =~ \(\#[0-9]+\)$ ]]; } -function is_empty_tree { git diff --quiet "$1"^ "$1"; } -function short_sha { - echo "${1:0:7}" -} - -############################################################################### -# args & fetch -############################################################################### -pr_head_ref=${1:-}; pr_base_ref=${2:-} -[[ -z $pr_head_ref || -z $pr_base_ref ]] && die "usage: $0 " - -git fetch origin "$pr_base_ref" "$pr_head_ref" -merge_base=$(git merge-base "origin/$pr_base_ref" "origin/$pr_head_ref") - -############################################################################### -# collect candidate commits -############################################################################### -commits=() -echo "📋 Collecting commits from $merge_base to origin/$pr_head_ref" >&2 -while read -r c; do - msg=$(git log -1 --format=%s "$c") - short=$(short_sha "$c") - if is_merge "$c"; then - echo " ⏭️ Skipping merge commit: $short - $msg" >&2 - continue - fi - if is_empty_tree "$c"; then - echo " ⏭️ Skipping empty commit: $short - $msg" >&2 - continue - fi - if has_pr_suffix "$c"; then - echo " ⏭️ Skipping PR commit: $short - $msg" >&2 - continue - fi - echo " ✅ Including commit: $short - $msg" >&2 - commits+=("$c") -done < <(git rev-list --reverse "$merge_base".."origin/$pr_head_ref") - -[[ ${#commits[@]} -eq 0 ]] && { echo "No commits to rebase" >&2; exit 0; } -total=${#commits[@]} -echo "📊 Total commits to rebase: $total" >&2 - -############################################################################### -# configure git based on most common author in candidate commits -############################################################################### -echo "🔧 Configuring git author from candidate commits" >&2 - -# Get the most common author from the candidate commits -author_info="" -if [[ ${#commits[@]} -gt 0 ]]; then - author_info=$( - for commit in "${commits[@]}"; do - git log -1 --format="%an|%ae" "$commit" - done | sort | uniq -c | sort -rn | head -1 | awk '{$1=""; print $0}' | xargs - ) -fi - -if [[ -n "$author_info" ]]; then - author_name=$(echo "$author_info" | cut -d'|' -f1) - author_email=$(echo "$author_info" | cut -d'|' -f2) - echo " Using most common author: $author_name <$author_email>" >&2 -else - # Fallback to aztec-bot - author_name="AztecBot" - author_email="tech@aztecprotocol.com" - echo " Using fallback author: $author_name <$author_email>" >&2 +# args +pr_head_ref=${1:-} +pr_base_ref=${2:-} +[[ -z $pr_head_ref || -z $pr_base_ref ]] && { echo "usage: $0 " >&2; exit 1; } + +merge_base=$(git merge-base "origin/$pr_base_ref" "$pr_head_ref") + +# collect commits, excluding empty commits, merge commits and previous squashed PR merges +commits=($( + git rev-list --reverse \ + --grep='\[empty\]' --grep='(#' \ + --no-merges \ + --invert-grep \ + "$merge_base..$pr_head_ref" +)) + +if [ ${#commits[@]} -eq 0 ]; then + echo "No commits to rebase" >&2 fi -git config user.name "$author_name" -git config user.email "$author_email" - -############################################################################### -# prepare working branch -############################################################################### -work_branch="auto-rebase-${pr_head_ref//\//-}" -git switch -c "$work_branch" "origin/$pr_base_ref" 2>/dev/null || { - git switch "$work_branch" - git reset --hard "origin/$pr_base_ref" -} - -############################################################################### -# helper functions -############################################################################### -# Try to apply a single commit using squash merge -function apply_single { - local sha="$1" before=$(git rev-parse --verify HEAD) - local short="$(short_sha "$sha")" - local msg=$(git log -1 --format=%s "$sha") - - echo "🔍 Trying single commit: $short - $msg" >&2 - - # Try to squash merge this single commit - if git merge --squash --no-commit "$sha" >/dev/null 2>&1; then - # Check if there are any changes - if git diff --cached --quiet; then - echo " ❌ No changes" >&2 - git reset --hard "$before" - return 1 - fi - - # Commit with the original commit's title - local original_msg=$(git log -1 --format=%s "$sha") - local original_body=$(git log -1 --format=%b "$sha") - if [[ -n "$original_body" ]]; then - git commit -m "$original_msg" -m "$original_body" >/dev/null - else - git commit -m "$original_msg" >/dev/null - fi - echo " ✅ Applied" >&2 - return 0 - else - echo " ❌ Conflict" >&2 - git merge --abort >/dev/null 2>&1 || true - git reset --hard "$before" - return 1 - fi -} - -# Create a squash commit for a range of commits -function create_squash_commit { - local start_idx=$1 end_idx=$2 - - # If it's just a single commit, preserve its original message - if [[ $start_idx -eq $end_idx ]]; then - local sha="${commits[$start_idx]}" - local original_msg=$(git log -1 --format=%s "$sha") - local original_body=$(git log -1 --format=%b "$sha") - if [[ -n "$original_body" ]]; then - git commit -m "$original_msg" -m "$original_body" >/dev/null - else - git commit -m "$original_msg" >/dev/null - fi - return - fi - - # Multiple commits - use squash format - links=() - commit_list="" - for ((k=start_idx; k<=end_idx; k++)); do - local sha="${commits[$k]}" - local short_sha="${sha:0:7}" - local msg=$(git log -1 --format=%s "$sha") - links+=("$short_sha") - commit_list="${commit_list}- ${short_sha} ${msg} (${short_sha})"$'\n' - done - - # Remove trailing newline - commit_list=${commit_list%$'\n'} - - # Commit with both comma-separated links and detailed list - git commit -m "squash: $(IFS=, ; echo "${links[*]}")" -m "$commit_list" >/dev/null -} - -# Try to find and apply the minimal batch that works -function find_minimal_batch { - local start_idx=$1 - local before=$(git rev-parse --verify HEAD) - - echo "🔄 Finding minimal batch starting from commit $start_idx..." >&2 - - # Try increasingly larger batches - for ((end_idx=start_idx; end_idx&2 - - # Reset to before state - git reset --hard "$before" >/dev/null 2>&1 - - # Try to cherry-pick all commits in this range - local all_applied=true - for ((k=start_idx; k<=end_idx; k++)); do - if ! git cherry-pick --no-commit "${commits[$k]}" >/dev/null 2>&1; then - all_applied=false - git cherry-pick --abort >/dev/null 2>&1 || true - break +# create working branch +work_branch="auto-rebase-$pr_head_ref" +git switch --force-create "$work_branch" "origin/$pr_base_ref" + +# cherry-pick each commit +for commit in "${commits[@]}"; do + echo "Cherry-picking $(git log -1 --format='%h %s' "$commit")" + if ! git cherry-pick -X theirs "$commit"; then + echo "Failed to cherry-pick $(git log -1 --format='%h %s' "$commit")" >&2 + + # Check if we're in an interactive terminal + if [ -t 0 ] && [ -t 1 ]; then + echo "" + echo "Cherry-pick conflict detected! Dropping into bash shell." + echo "You can:" + echo " - Fix conflicts and run: git cherry-pick --continue" + echo " - Skip this commit: git cherry-pick --skip" + echo " - Abort and exit: git cherry-pick --abort && exit" + echo " - Exit shell to abort the rebase" + echo "" + + # Drop into interactive bash + bash || true + + # Check if cherry-pick is still in progress + if git rev-parse --verify CHERRY_PICK_HEAD >/dev/null 2>&1; then + echo "Cherry-pick still in progress, aborting..." + git cherry-pick --abort 2>/dev/null || true + git switch - + git branch -D "$work_branch" + exit 1 fi - done - - if $all_applied && ! git diff --cached --quiet; then - # Success! Create the squash commit - echo " ✅ Batch [$start_idx..$end_idx] works!" >&2 - create_squash_commit "$start_idx" "$end_idx" - echo $((end_idx + 1)) # Output next index to process - return 0 - fi - - # Reset for next attempt - git reset --hard "$before" >/dev/null 2>&1 - done - - # If we get here, we couldn't find a working batch - return 1 -} - -############################################################################### -# main rebasing loop -############################################################################### -idx=0 -while (( idx < total )); do - # First, always try to apply as a single commit - if apply_single "${commits[$idx]}"; then - ((idx++)) || true - continue - fi - - # Single commit failed. Find the minimal batch that works - if new_idx=$(find_minimal_batch "$idx"); then - idx=$new_idx - else - # Can't find any working batch. Last resort: squash all remaining - echo "⚠️ Cannot find working batch. Squashing all remaining commits..." >&2 - - # Reset and try to squash merge all remaining commits at once - before=$(git rev-parse --verify HEAD) - - # Create a temp branch at the last commit - temp_branch="temp-squash-remaining-$$" - git branch "$temp_branch" "${commits[$((total-1))]}" >/dev/null 2>&1 - - if git merge --squash "$temp_branch" >/dev/null 2>&1 && ! git diff --cached --quiet; then - create_squash_commit "$idx" "$((total-1))" - git branch -D "$temp_branch" >/dev/null 2>&1 - break else - git merge --abort >/dev/null 2>&1 || true - git branch -D "$temp_branch" >/dev/null 2>&1 || true - git reset --hard "$before" - - # Ultimate fallback: squash entire PR - echo "❌ Even remaining commits failed. Squashing entire PR..." >&2 - git reset --hard "origin/$pr_base_ref" - - if git merge --squash "origin/$pr_head_ref" >/dev/null 2>&1; then - create_squash_commit 0 "$((total-1))" - else - git merge --abort >/dev/null 2>&1 || true - die "rebase failed: unable to squash merge entire PR branch" - fi + git cherry-pick --abort 2>/dev/null || true + git switch - + git branch -D "$work_branch" + exit 1 fi - break fi done -############################################################################### -# fast-forward PR branch & optional push -############################################################################### -git branch -f "$pr_head_ref" HEAD -git switch "$pr_head_ref" -git branch -D "$work_branch" - -if [[ -n "${GH_TOKEN:-}" || -n "${GITHUB_TOKEN:-}" ]]; then +if [ "${PUSH:-0}" -eq 1 ]; then + # update PR branch + git switch --force-create "$pr_head_ref" "origin/$pr_base_ref" + git git reset --hard "$work_branch" git push origin "$pr_head_ref" --force-with-lease +else + echo "You are on a successful rebase branch. Use 'git log' to look around." + echo "To return, just do 'git checkout $pr_head_ref'." + echo "If you like what you see you can do:" + echo git reset --hard "$work_branch" + echo git push --force-with-lease fi - -echo "✅ Successfully rebased $pr_head_ref" >&2