feat: merge-train/barretenberg

.github/workflows/auto-rebase-pr.yml

@@ -30,9 +30,18 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
         run: |
+          set -x
           pr_head_ref=${{ github.event.pull_request.head.ref }}
           pr_base_ref=${{ github.event.pull_request.base.ref }}
-          ./scripts/auto_rebase_pr.sh "$pr_head_ref" "$pr_base_ref"
+          # ensure we have up to date head/base
+          git fetch origin "$pr_base_ref" "$pr_head_ref"
+          # Be sure to run the auto rebase script such that we don't write to it while it's being read.
+          cp ./scripts/auto_rebase_pr.sh .github/temp-script.sh
+          bash .github/temp-script.sh "$pr_head_ref" "$pr_base_ref"
+          rebase_sha=$(git rev-parse HEAD)
+          git switch --force-create "$pr_head_ref" "origin/$pr_base_ref"
+          git reset --hard "$rebase_sha"
+          git push origin "$pr_head_ref" --force-with-lease
 
       - name: Post failure comment
         if: failure() || steps.rebase.outcome == 'failure'

.github/workflows/ci3.yml

@@ -150,6 +150,7 @@ jobs:
           output-file-path: ./bench-out/bench.json
           github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
           auto-push: true
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
           alert-threshold: "105%"
           comment-on-alert: false
           fail-on-alert: false

.vscode/launch.json

@@ -36,13 +36,23 @@
       "name": "Debug CIVC transaction",
       "type": "lldb",
       "request": "launch",
-      "program": "${workspaceFolder}/barretenberg/cpp/build-debug/bin/bb",
+      "program": "${workspaceFolder}/barretenberg/cpp/build-debug-no-avm/bin/bb",
       "args": ["prove", "--scheme", "client_ivc", "--output_path", ".", "--ivc_inputs_path", "ivc-inputs.msgpack"],
       "cwd": "${workspaceFolder}/yarn-project/end-to-end/example-app-ivc-inputs-out/ecdsar1+transfer_0_recursions+sponsored_fpc",
       "initCommands": [
         "command script import ${workspaceFolder}/barretenberg/cpp/scripts/lldb_format.py"
       ],
     },
+    {
+      "name": "Debug Commitment Schemes Tests",
+      "type": "lldb",
+      "request": "launch",
+      "program": "${workspaceFolder}/barretenberg/cpp/build-debug/bin/commitment_schemes_tests",
+      "args": [],
+      "initCommands": [
+        "command script import ${workspaceFolder}/barretenberg/cpp/scripts/lldb_format.py"
+      ],
+    },
     {
       "name": "Debug BB API Test",
       "type": "lldb",

barretenberg/cpp/bootstrap.sh

@@ -308,11 +308,9 @@ case "$cmd" in
 
     # Download cached flow inputs from the specified commit
     export AZTEC_CACHE_COMMIT=$commit_hash
+    # TODO currently does nothing! to reinstate in cache_download
     export FORCE_CACHE_DOWNLOAD=${FORCE_CACHE_DOWNLOAD:-1}
-    echo "Running with FORCE_CACHE_DOWNLOAD=1. This should work, but as a workaround you can set FORCE_CACHE_DOWNLOAD=0 before the bench_ivc call to build some parts."
-    ../../noir/bootstrap.sh
-    USE_CIRCUITS_CACHE=1 ../../noir-projects/noir-protocol-circuits/bootstrap.sh
-    yarn --cwd ../../yarn-project/bb-prover generate
+    USE_CIRCUITS_CACHE=1 BOOTSTRAP_AFTER=barretenberg BOOSTRAP_TO=yarn-project ../../bootstrap.sh
 
     rm -rf bench-out
 

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -11,7 +11,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-c8814328.tar.gz"
+pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-79e2c5e0.tar.gz"
 
 # For easily rerunning the inputs generation
 if [[ "${1:-}" == "--update_inputs" ]]; then

barretenberg/cpp/src/barretenberg/api/api.hpp

@@ -24,6 +24,7 @@ class API {
                                    // recursive verifier) or is it for an ivc verifier?
         bool write_vk{ false };    // should we addditionally write the verification key when writing the proof
         bool include_gates_per_opcode{ false }; // should we include gates_per_opcode in the gates command output
+        bool slow_low_memory{ false };          // use file backed memory for polynomials
 
         friend std::ostream& operator<<(std::ostream& os, const Flags& flags)
         {
@@ -42,6 +43,7 @@ class API {
                << "  verifier_type: " << flags.verifier_type << "\n"
                << "  write_vk " << flags.write_vk << "\n"
                << "  include_gates_per_opcode " << flags.include_gates_per_opcode << "\n"
+               << "  slow_low_memory " << flags.slow_low_memory << "\n"
                << "]" << std::endl;
             return os;
         }

barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp

@@ -1,5 +1,7 @@
 #include "api_client_ivc.hpp"
+#include "barretenberg/api/bbapi.hpp"
 #include "barretenberg/api/file_io.hpp"
+#include "barretenberg/api/get_bytecode.hpp"
 #include "barretenberg/api/log.hpp"
 #include "barretenberg/api/write_prover_output.hpp"
 #include "barretenberg/client_ivc/client_ivc.hpp"
@@ -13,37 +15,11 @@
 #include "barretenberg/serialize/msgpack.hpp"
 #include "barretenberg/serialize/msgpack_check_eq.hpp"
 #include <algorithm>
+#include <sstream>
 #include <stdexcept>
 
 namespace bb {
-
-acir_format::WitnessVector witness_map_to_witness_vector(std::map<std::string, std::string> const& witness_map)
-{
-    acir_format::WitnessVector wv;
-    size_t index = 0;
-    for (auto& e : witness_map) {
-        uint64_t value = stoull(e.first);
-        // ACIR uses a sparse format for WitnessMap where unused witness indices may be left unassigned.
-        // To ensure that witnesses sit at the correct indices in the `WitnessVector`, we fill any indices
-        // which do not exist within the `WitnessMap` with the dummy value of zero.
-        while (index < value) {
-            wv.push_back(fr(0));
-            index++;
-        }
-        wv.push_back(fr(uint256_t(e.second)));
-        index++;
-    }
-    return wv;
-}
-
-std::shared_ptr<ClientIVC::DeciderProvingKey> get_acir_program_decider_proving_key(acir_format::AcirProgram& program)
-{
-    ClientIVC::ClientCircuit builder = acir_format::create_circuit<ClientIVC::ClientCircuit>(program);
-
-    // Construct the verification key via the prover-constructed proving key with the proper trace settings
-    TraceSettings trace_settings{ AZTEC_TRACE_STRUCTURE };
-    return std::make_shared<ClientIVC::DeciderProvingKey>(builder, trace_settings);
-}
+namespace { // anonymous namespace
 
 /**
  * @brief Compute and write to file a MegaHonk VK for a circuit to be accumulated in the IVC
@@ -53,52 +29,57 @@ std::shared_ptr<ClientIVC::DeciderProvingKey> get_acir_program_decider_proving_k
  * @param bytecode_path
  * @param witness_path
  */
-void write_standalone_vk(const std::string& output_data_type,
-                         const std::string& bytecode_path,
-                         const std::string& output_path)
+void write_standalone_vk(const std::string& output_format,
+                         const std::filesystem::path& bytecode_path,
+                         const std::filesystem::path& output_path)
 {
-
-    acir_format::AcirProgram program{ get_constraint_system(bytecode_path), /*witness=*/{} };
-    std::shared_ptr<ClientIVC::DeciderProvingKey> proving_key = get_acir_program_decider_proving_key(program);
-    auto verification_key = std::make_shared<ClientIVC::MegaVerificationKey>(proving_key->get_precomputed());
-    PubInputsProofAndKey<ClientIVC::MegaVerificationKey> to_write{ .key = verification_key };
-
-    write(to_write, output_data_type, "vk", output_path);
-}
-
-size_t get_num_public_inputs_in_circuit(const std::filesystem::path& bytecode_path)
-{
-    using namespace acir_format;
-    acir_format::AcirProgram program{ get_constraint_system(bytecode_path), /*witness=*/{} };
-    return program.constraints.public_inputs.size();
+    auto bytecode = get_bytecode(bytecode_path);
+    auto response = bbapi::ClientIvcComputeStandaloneVk{
+        .circuit = { .name = "standalone_circuit", .bytecode = std::move(bytecode) }
+    }.execute();
+
+    bool wrote_file = false;
+    bool is_stdout = output_path == "-";
+    auto write_fn = [&](const std::filesystem::path& path, const auto& data) {
+        if (is_stdout) {
+            write_bytes_to_stdout(data);
+        } else {
+            write_file(path, data);
+        }
+    };
+    if (output_format == "bytes_and_fields" && is_stdout) {
+        throw_or_abort("Cannot write to stdout in bytes_and_fields format.");
+    }
+    if (output_format == "bytes" || output_format == "bytes_and_fields") {
+        write_fn(output_path / "vk", response.bytes);
+        wrote_file = true;
+    }
+    if (output_format == "fields" || output_format == "bytes_and_fields") {
+        std::string json = field_elements_to_json(response.fields);
+        write_fn(output_path / "vk_fields.json", std::vector<uint8_t>(json.begin(), json.end()));
+        wrote_file = true;
+    }
+    if (!wrote_file) {
+        throw_or_abort("Unsupported output format for standalone vk: " + output_format);
+    }
 }
 
-void write_vk_for_ivc(const std::string& output_format,
-                      size_t num_public_inputs_in_final_circuit,
-                      const std::filesystem::path& output_dir)
+void write_civc_vk(const std::string& output_format,
+                   size_t num_public_inputs_in_final_circuit,
+                   const std::filesystem::path& output_dir)
 {
     if (output_format != "bytes") {
         throw_or_abort("Unsupported output format for ClientIVC vk: " + output_format);
     }
-    ClientIVC ivc{ /*num_circuits=*/2, { AZTEC_TRACE_STRUCTURE } };
-    ClientIVCMockCircuitProducer circuit_producer;
-
-    // Initialize the IVC with an arbitrary circuit
-    // We segfault if we only call accumulate once
-    static constexpr size_t SMALL_ARBITRARY_LOG_CIRCUIT_SIZE{ 5 };
-    MegaCircuitBuilder circuit_0 = circuit_producer.create_next_circuit(ivc, SMALL_ARBITRARY_LOG_CIRCUIT_SIZE);
-    ivc.accumulate(circuit_0);
-
-    // Create another circuit and accumulate
-    MegaCircuitBuilder circuit_1 =
-        circuit_producer.create_next_circuit(ivc, SMALL_ARBITRARY_LOG_CIRCUIT_SIZE, num_public_inputs_in_final_circuit);
-    ivc.accumulate(circuit_1);
 
-    // Construct the hiding circuit and its VK (stored internally in the IVC)
-    ivc.construct_hiding_circuit_key();
+    // Since we need to specify the number of public inputs but ClientIvcComputeIvcVk derives it from bytecode,
+    // we need to create a mock circuit with the correct number of public inputs
+    // For now, we'll use the compute_civc_vk function directly as it was designed for this purpose
+    bbapi::BBApiRequest request;
+    auto vk = bbapi::compute_civc_vk(request, num_public_inputs_in_final_circuit);
+    const auto buf = to_buffer(vk);
 
     const bool output_to_stdout = output_dir == "-";
-    const auto buf = to_buffer(ivc.get_vk());
 
     if (output_to_stdout) {
         write_bytes_to_stdout(buf);
@@ -107,33 +88,53 @@ void write_vk_for_ivc(const std::string& output_format,
     }
 }
 
-void write_vk_for_ivc(const std::string& output_data_type,
-                      const std::string& bytecode_path,
-                      const std::filesystem::path& output_dir)
+void write_civc_vk(const std::string& output_data_type,
+                   const std::string& bytecode_path,
+                   const std::filesystem::path& output_dir)
 {
-    const size_t num_public_inputs_in_final_circuit = get_num_public_inputs_in_circuit(bytecode_path);
-    info("num_public_inputs_in_final_circuit: ", num_public_inputs_in_final_circuit);
-    write_vk_for_ivc(output_data_type, num_public_inputs_in_final_circuit, output_dir);
+    if (output_data_type != "bytes") {
+        throw_or_abort("Unsupported output format for ClientIVC vk: " + output_data_type);
+    }
+
+    auto bytecode = get_bytecode(bytecode_path);
+
+    auto response = bbapi::ClientIvcComputeIvcVk{
+        .circuit = { .name = "final_circuit", .bytecode = std::move(bytecode) }
+    }.execute();
+
+    const bool output_to_stdout = output_dir == "-";
+    if (output_to_stdout) {
+        write_bytes_to_stdout(response.bytes);
+    } else {
+        write_file(output_dir / "vk", response.bytes);
+    }
 }
+} // anonymous namespace
 
 void ClientIVCAPI::prove(const Flags& flags,
                          const std::filesystem::path& input_path,
                          const std::filesystem::path& output_dir)
 {
 
-    PrivateExecutionSteps steps;
-    steps.parse(PrivateExecutionStepRaw::load_and_decompress(input_path));
+    bbapi::BBApiRequest request;
+    std::vector<PrivateExecutionStepRaw> raw_steps = PrivateExecutionStepRaw::load_and_decompress(input_path);
 
-    std::shared_ptr<ClientIVC> ivc = steps.accumulate();
-    ClientIVC::Proof proof = ivc->prove();
+    bbapi::ClientIvcStart{ .num_circuits = raw_steps.size() }.execute(request);
+
+    size_t loaded_circuit_public_inputs_size = 0;
+    for (const auto& step : raw_steps) {
+        bbapi::ClientIvcLoad{
+            .circuit = { .name = step.function_name, .bytecode = step.bytecode, .verification_key = step.vk }
+        }.execute(request);
 
-    // We verify this proof. Another bb call to verify has the overhead of loading the SRS,
-    // and it is mysterious if this transaction fails later in the lifecycle.
-    // The files are still written in case they are needed to investigate this failure.
-    if (!ivc->verify(proof)) {
-        THROW std::runtime_error("Failed to verify the private (ClientIVC) transaction proof!");
+        // NOLINTNEXTLINE(bugprone-unchecked-optional-access): we know the optional has been set here.
+        loaded_circuit_public_inputs_size = request.loaded_circuit_constraints->public_inputs.size();
+        info("ClientIVC: accumulating " + step.function_name);
+        bbapi::ClientIvcAccumulate{ .witness = step.witness }.execute(request);
     }
 
+    auto proof = bbapi::ClientIvcProve{}.execute(request).proof;
+
     // We'd like to use the `write` function that UltraHonkAPI uses, but there are missing functions for creating
     // std::string representations of vks that don't feel worth implementing
     const bool output_to_stdout = output_dir == "-";
@@ -153,8 +154,7 @@ void ClientIVCAPI::prove(const Flags& flags,
 
     if (flags.write_vk) {
         vinfo("writing ClientIVC vk in directory ", output_dir);
-        const size_t num_public_inputs_in_final_circuit = steps.folding_stack.back().constraints.public_inputs.size();
-        write_vk_for_ivc("bytes", num_public_inputs_in_final_circuit, output_dir);
+        write_civc_vk("bytes", loaded_circuit_public_inputs_size, output_dir);
     }
 }
 
@@ -194,21 +194,22 @@ void ClientIVCAPI::write_solidity_verifier([[maybe_unused]] const Flags& flags,
 }
 
 bool ClientIVCAPI::check_precomputed_vks(const std::filesystem::path& input_path)
-
 {
-    PrivateExecutionSteps steps;
-    steps.parse(PrivateExecutionStepRaw::load_and_decompress(input_path));
+    bbapi::BBApiRequest request;
+    std::vector<PrivateExecutionStepRaw> raw_steps = PrivateExecutionStepRaw::load_and_decompress(input_path);
 
-    for (auto [program, precomputed_vk, function_name] :
-         zip_view(steps.folding_stack, steps.precomputed_vks, steps.function_names)) {
-        if (precomputed_vk == nullptr) {
-            info("FAIL: Expected precomputed vk for function ", function_name);
+    for (const auto& step : raw_steps) {
+        if (step.vk.empty()) {
+            info("FAIL: Expected precomputed vk for function ", step.function_name);
             return false;
         }
-        std::shared_ptr<ClientIVC::DeciderProvingKey> proving_key = get_acir_program_decider_proving_key(program);
-        auto computed_vk = std::make_shared<ClientIVC::MegaVerificationKey>(proving_key->get_precomputed());
-        std::string error_message = "FAIL: Precomputed vk does not match computed vk for function " + function_name;
-        if (!msgpack::msgpack_check_eq(*computed_vk, *precomputed_vk, error_message)) {
+        auto response = bbapi::ClientIvcCheckPrecomputedVk{ .circuit = { .name = step.function_name,
+                                                                         .bytecode = step.bytecode,
+                                                                         .verification_key = step.vk },
+                                                            .function_name = step.function_name }
+                            .execute();
+
+        if (!response.valid) {
             return false;
         }
     }
@@ -221,7 +222,7 @@ void ClientIVCAPI::write_vk(const Flags& flags,
 {
 
     if (flags.verifier_type == "ivc") {
-        write_vk_for_ivc(flags.output_format, bytecode_path, output_path);
+        write_civc_vk(flags.output_format, bytecode_path, output_path);
     } else if (flags.verifier_type == "standalone") {
         write_standalone_vk(flags.output_format, bytecode_path, output_path);
     } else {

barretenberg/cpp/src/barretenberg/api/api_client_ivc.hpp

@@ -44,8 +44,6 @@ void gate_count_for_ivc(const std::string& bytecode_path, bool include_gates_per
 
 void write_arbitrary_valid_client_ivc_proof_and_vk_to_file(const std::filesystem::path& output_dir);
 
-acir_format::WitnessVector witness_map_to_witness_vector(std::map<std::string, std::string> const& witness_map);
-
 std::vector<uint8_t> decompress(const void* bytes, size_t size);
 
 } // namespace bb