From f5378ee735cbf601aa71214a70e4d34b595cd456 Mon Sep 17 00:00:00 2001 From: Suyash Bagad Date: Tue, 11 Apr 2023 15:25:29 +0000 Subject: [PATCH 01/92] Regenerate pedersen lookup tables if they're empty --- .../barretenberg/crypto/pedersen_hash/pedersen_lookup.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/crypto/pedersen_hash/pedersen_lookup.cpp b/cpp/src/barretenberg/crypto/pedersen_hash/pedersen_lookup.cpp index f455c684be..e5a6fd9884 100644 --- a/cpp/src/barretenberg/crypto/pedersen_hash/pedersen_lookup.cpp +++ b/cpp/src/barretenberg/crypto/pedersen_hash/pedersen_lookup.cpp @@ -66,7 +66,11 @@ void init() { ASSERT(BITS_PER_TABLE < BITS_OF_BETA); ASSERT(BITS_PER_TABLE + BITS_OF_BETA < BITS_ON_CURVE); - if (inited) { + bool pedersen_tables_filled = pedersen_iv_table.size() > 0; + for (size_t i = 0; i < NUM_PEDERSEN_TABLES; i++) { + pedersen_tables_filled = (pedersen_tables_filled && (pedersen_tables[i].size() > 0)); + } + if (inited && pedersen_tables_filled) { return; } generators = grumpkin::g1::derive_generators(); From 6b4eaa9e5474de9af1e893b94e1b47c28a5e83ec Mon Sep 17 00:00:00 2001 From: Suyash Bagad Date: Tue, 11 Apr 2023 15:28:55 +0000 Subject: [PATCH 02/92] re-init generator tables if they're empty. --- cpp/src/barretenberg/crypto/generators/generator_data.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/crypto/generators/generator_data.cpp b/cpp/src/barretenberg/crypto/generators/generator_data.cpp index 7970dfe659..3c07a4348a 100644 --- a/cpp/src/barretenberg/crypto/generators/generator_data.cpp +++ b/cpp/src/barretenberg/crypto/generators/generator_data.cpp @@ -205,7 +205,7 @@ const fixed_base_ladder* get_ladder_internal(std::array> const& init_generator_data() { static std::vector> global_generator_data; - if (inited) { + if (inited && (global_generator_data.size() == size_of_generator_data_array)) { return global_generator_data; } std::vector generators; From e884272e088a072f7ba36f6bac7ae8a253dc7484 Mon Sep 17 00:00:00 2001 From: Maddiaa <47148561+cheethas@users.noreply.github.com> Date: Tue, 18 Apr 2023 05:12:08 -0700 Subject: [PATCH 03/92] feat(nullifier_tree): make empty nullifier tree leaves hash be 0 (#360) * feat(nullifier_tree): make empty nullifier tree leaves be 0 * fix: add append zero behaviour and test * fix: explicit type name * clean: update class semantics --------- Co-authored-by: cheethas Co-authored-by: cheethas --- .../nullifier_tree/nullifier_leaf.hpp | 69 +++++- .../nullifier_tree/nullifier_memory_tree.cpp | 34 ++- .../nullifier_tree/nullifier_memory_tree.hpp | 10 +- .../nullifier_memory_tree.test.cpp | 216 ++++++++++++++++-- .../nullifier_tree/nullifier_tree.cpp | 23 +- .../nullifier_tree/nullifier_tree.hpp | 2 +- 6 files changed, 304 insertions(+), 50 deletions(-) diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp index bac3478e6a..d7080b48bb 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp @@ -40,14 +40,79 @@ struct nullifier_leaf { barretenberg::fr hash() const { return stdlib::merkle_tree::hash_multiple_native({ value, nextIndex, nextValue }); } }; -inline std::pair find_closest_leaf(std::vector const& leaves_, fr const& new_value) +/** + * @brief Wrapper for the Nullifier leaf class that allows for 0 values + * + */ +class WrappedNullifierLeaf { + + public: + // Initialise with a nullifier leaf + WrappedNullifierLeaf(nullifier_leaf value) + : data(value) + {} + // Initialise an empty leaf + WrappedNullifierLeaf() + : data(std::nullopt) + {} + + bool operator==(WrappedNullifierLeaf const&) const = default; + + /** + * @brief Pass through the underlying std::optional method + * + * @return true + * @return false + */ + bool has_value() const { return data.has_value(); } + + /** + * @brief Return the wrapped nullifier_leaf object + * + * @return nullifier_leaf + */ + nullifier_leaf unwrap() const { return data.value(); } + + /** + * @brief Set the wrapped nullifier_leaf object value + * + * @param value + */ + void set(nullifier_leaf value) { data.emplace(value); } + + /** + * @brief Return the hash of the wrapped object, other return the zero hash of 0 + * + * @return barretenberg::fr + */ + barretenberg::fr hash() const { return data.has_value() ? data.value().hash() : barretenberg::fr::zero(); } + + /** + * @brief Generate a zero leaf (call the constructor with no arguments) + * + * @return NullifierLeaf + */ + static WrappedNullifierLeaf zero() { return WrappedNullifierLeaf(); } + + private: + // Underlying data + std::optional data; +}; + +inline std::pair find_closest_leaf(std::vector const& leaves_, fr const& new_value) { std::vector diff; bool repeated = false; auto new_value_ = uint256_t(new_value); for (size_t i = 0; i < leaves_.size(); i++) { - auto leaf_value_ = uint256_t(leaves_[i].value); + + if (!leaves_[i].has_value()) { + diff.push_back(new_value_); + continue; + } + + auto leaf_value_ = uint256_t(leaves_[i].unwrap().value); if (leaf_value_ > new_value_) { diff.push_back(leaf_value_); } else if (leaf_value_ == new_value_) { diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp index 09398f5a5a..f070ab2ded 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp @@ -12,11 +12,8 @@ NullifierMemoryTree::NullifierMemoryTree(size_t depth) total_size_ = 1UL << depth_; hashes_.resize(total_size_ * 2 - 2); - // Build the entire tree. - nullifier_leaf zero_leaf = { 0, 0, 0 }; - leaves_.push_back(zero_leaf); - auto current = zero_leaf.hash(); - update_element(0, current); + // Build the entire tree and fill with 0 hashes. + auto current = WrappedNullifierLeaf::zero().hash(); size_t layer_size = total_size_; for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { for (size_t i = 0; i < layer_size; ++i) { @@ -25,30 +22,45 @@ NullifierMemoryTree::NullifierMemoryTree(size_t depth) current = hash_pair_native(current, current); } - root_ = current; + // Insert the initial leaf at index 0 + auto initial_leaf = WrappedNullifierLeaf(nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }); + leaves_.push_back(initial_leaf); + root_ = update_element(0, initial_leaf.hash()); } fr NullifierMemoryTree::update_element(fr const& value) { // Find the leaf with the value closest and less than `value` + + // If value is 0 we simply append 0 a null NullifierLeaf to the tree + if (value == 0) { + auto zero_leaf = WrappedNullifierLeaf::zero(); + leaves_.push_back(zero_leaf); + return update_element(leaves_.size() - 1, zero_leaf.hash()); + } + size_t current; bool is_already_present; std::tie(current, is_already_present) = find_closest_leaf(leaves_, value); + nullifier_leaf current_leaf = leaves_[current].unwrap(); nullifier_leaf new_leaf = { .value = value, - .nextIndex = leaves_[current].nextIndex, - .nextValue = leaves_[current].nextValue }; + .nextIndex = current_leaf.nextIndex, + .nextValue = current_leaf.nextValue }; + if (!is_already_present) { // Update the current leaf to point it to the new leaf - leaves_[current].nextIndex = leaves_.size(); - leaves_[current].nextValue = value; + current_leaf.nextIndex = leaves_.size(); + current_leaf.nextValue = value; + + leaves_[current].set(current_leaf); // Insert the new leaf with (nextIndex, nextValue) of the current leaf leaves_.push_back(new_leaf); } // Update the old leaf in the tree - auto old_leaf_hash = leaves_[current].hash(); + auto old_leaf_hash = current_leaf.hash(); size_t old_leaf_index = current; auto root = update_element(old_leaf_index, old_leaf_hash); diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp index 3a41d69194..d40b2ea2bc 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp @@ -68,6 +68,7 @@ using namespace barretenberg; * nextVal 10 50 20 30 0 0 0 0 */ class NullifierMemoryTree : public MemoryTree { + public: NullifierMemoryTree(size_t depth); @@ -78,15 +79,18 @@ class NullifierMemoryTree : public MemoryTree { fr update_element(fr const& value); const std::vector& get_hashes() { return hashes_; } - const std::vector& get_leaves() { return leaves_; } - const nullifier_leaf& get_leaf(size_t index) { return leaves_[index]; } + const WrappedNullifierLeaf get_leaf(size_t index) + { + return (index < leaves_.size()) ? leaves_[index] : WrappedNullifierLeaf::zero(); + } + const std::vector& get_leaves() { return leaves_; } protected: using MemoryTree::depth_; using MemoryTree::hashes_; using MemoryTree::root_; using MemoryTree::total_size_; - std::vector leaves_; + std::vector leaves_; }; } // namespace merkle_tree diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp index b404519fd4..1a5b5bfae8 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp @@ -62,8 +62,8 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(30); EXPECT_EQ(tree.get_leaves().size(), 2); - EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); /** * Add new value 10: @@ -76,9 +76,9 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(10); EXPECT_EQ(tree.get_leaves().size(), 3); - EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 1, 30 }).hash()); /** * Add new value 20: @@ -91,18 +91,18 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), nullifier_leaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); // Adding the same value must not affect anything tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), nullifier_leaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); /** * Add new value 50: @@ -115,11 +115,11 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(50); EXPECT_EQ(tree.get_leaves().size(), 5); - EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 4, 50 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), nullifier_leaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[4].hash(), nullifier_leaf({ 50, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 4, 50 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf({ 50, 0, 0 }).hash()); // Manually compute the node values auto e000 = tree.get_leaves()[0].hash(); @@ -127,9 +127,9 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) auto e010 = tree.get_leaves()[2].hash(); auto e011 = tree.get_leaves()[3].hash(); auto e100 = tree.get_leaves()[4].hash(); - auto e101 = nullifier_leaf({ 0, 0, 0 }).hash(); - auto e110 = nullifier_leaf({ 0, 0, 0 }).hash(); - auto e111 = nullifier_leaf({ 0, 0, 0 }).hash(); + auto e101 = WrappedNullifierLeaf::zero().hash(); + auto e110 = WrappedNullifierLeaf::zero().hash(); + auto e111 = WrappedNullifierLeaf::zero().hash(); auto e00 = hash_pair_native(e000, e001); auto e01 = hash_pair_native(e010, e011); @@ -161,6 +161,172 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) EXPECT_EQ(tree.get_hash_path(7), expected); } +TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) +{ + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + NullifierMemoryTree tree(depth); + + /** + * Intial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 0 0 0 0 0 0 0 + * nextIdx 0 0 0 0 0 0 0 0 + * nextVal 0 0 0 0 0 0 0 0 + */ + WrappedNullifierLeaf zero_leaf = WrappedNullifierLeaf({ 0, 0, 0 }); + EXPECT_EQ(tree.get_leaves().size(), 1); + EXPECT_EQ(tree.get_leaves()[0], zero_leaf); + + /** + * Add new value 30: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 30 0 0 0 0 0 0 0 + */ + tree.update_element(30); + EXPECT_EQ(tree.get_leaves().size(), 2); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + + /** + * Add new value 10: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 0 0 0 0 0 + * nextIdx 2 0 1 0 0 0 0 0 + * nextVal 10 0 30 0 0 0 0 0 + */ + tree.update_element(10); + EXPECT_EQ(tree.get_leaves().size(), 3); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 1, 30 }).hash()); + + /** + * Add new value 20: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + */ + tree.update_element(20); + EXPECT_EQ(tree.get_leaves().size(), 4); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + + // Adding the same value must not affect anything + tree.update_element(20); + EXPECT_EQ(tree.get_leaves().size(), 4); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + + /** + * Add new value 0: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + */ + tree.update_element(0); + EXPECT_EQ(tree.get_leaves().size(), 5); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf::zero().hash()); + + /* + * Add new value 0: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + */ + tree.update_element(0); + EXPECT_EQ(tree.get_leaves().size(), 6); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves()[5].hash(), WrappedNullifierLeaf::zero().hash()); + + /** + * Add new value 50: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 50 0 + * nextIdx 2 6 3 1 0 0 0 0 + * nextVal 10 50 20 30 0 0 0 0 + */ + tree.update_element(50); + EXPECT_EQ(tree.get_leaves().size(), 7); + EXPECT_EQ(tree.get_leaf(0).hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaf(1).hash(), WrappedNullifierLeaf({ 30, 6, 50 }).hash()); + EXPECT_EQ(tree.get_leaf(2).hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaf(3).hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaf(4).hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaf(5).hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaf(6).hash(), WrappedNullifierLeaf({ 50, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaf(7).hash(), WrappedNullifierLeaf::zero().hash()); + + // Manually compute the node values + auto e000 = tree.get_leaf(0).hash(); + auto e001 = tree.get_leaf(1).hash(); + auto e010 = tree.get_leaf(2).hash(); + auto e011 = tree.get_leaf(3).hash(); + auto e100 = tree.get_leaf(4).hash(); + auto e101 = tree.get_leaf(5).hash(); + auto e110 = tree.get_leaf(6).hash(); + auto e111 = tree.get_leaf(7).hash(); + + auto e00 = hash_pair_native(e000, e001); + auto e01 = hash_pair_native(e010, e011); + auto e10 = hash_pair_native(e100, e101); + auto e11 = hash_pair_native(e110, e111); + + auto e0 = hash_pair_native(e00, e01); + auto e1 = hash_pair_native(e10, e11); + auto root = hash_pair_native(e0, e1); + + // Check the hash path at index 2 and 3 + // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) + fr_hash_path expected = { + std::make_pair(e010, e011), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(2), expected); + EXPECT_EQ(tree.get_hash_path(3), expected); + EXPECT_EQ(tree.root(), root); + + // Check the hash path at index 6 and 7 + expected = { + std::make_pair(e110, e111), + std::make_pair(e10, e11), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(6), expected); + EXPECT_EQ(tree.get_hash_path(7), expected); +} TEST(crypto_nullifier_tree, test_nullifier_tree) { // Create a depth-8 indexed merkle tree @@ -190,8 +356,10 @@ TEST(crypto_nullifier_tree, test_nullifier_tree) const auto& leaves = tree.get_leaves(); std::vector differences; for (size_t i = 0; i < leaves.size(); i++) { - uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(leaves[i].value)); - uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(leaves[i].nextValue)); + uint256_t diff_hi = + abs_diff(uint256_t(new_member), uint256_t(leaves[i].has_value() ? leaves[i].unwrap().value : 0)); + uint256_t diff_lo = + abs_diff(uint256_t(new_member), uint256_t(leaves[i].has_value() ? leaves[i].unwrap().nextValue : 0)); differences.push_back(diff_hi + diff_lo); } auto it = std::min_element(differences.begin(), differences.end()); @@ -199,5 +367,5 @@ TEST(crypto_nullifier_tree, test_nullifier_tree) // Merkle proof at `index` proves non-membership of `new_member` auto hash_path = tree.get_hash_path(index); - EXPECT_TRUE(check_hash_path(tree.root(), hash_path, leaves[index], index)); + EXPECT_TRUE(check_hash_path(tree.root(), hash_path, leaves[index].unwrap(), index)); } \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp index 0a36b8020c..0280670f15 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp @@ -29,10 +29,13 @@ NullifierTree::NullifierTree(Store& store, size_t depth, uint8_t tree_id) // Compute the zero values at each layer. // Insert the zero leaf to the `leaves` and also to the tree at index 0. - auto zero_leaf = nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }; - leaves.push_back(zero_leaf); - auto current = zero_leaf.hash(); - update_element(0, current); + WrappedNullifierLeaf initial_leaf = + WrappedNullifierLeaf(nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }); + leaves.push_back(initial_leaf); + update_element(0, initial_leaf.hash()); + + // Create the zero hashes for the tree + auto current = WrappedNullifierLeaf::zero().hash(); for (size_t i = 0; i < depth; ++i) { zero_hashes_[i] = current; current = hash_pair_native(current, current); @@ -53,13 +56,15 @@ template fr NullifierTree::update_element(fr const& valu bool is_already_present; std::tie(current, is_already_present) = find_closest_leaf(leaves, value); - nullifier_leaf new_leaf = { .value = value, - .nextIndex = leaves[current].nextIndex, - .nextValue = leaves[current].nextValue }; + nullifier_leaf current_leaf = leaves[current].unwrap(); + WrappedNullifierLeaf new_leaf = WrappedNullifierLeaf( + { .value = value, .nextIndex = current_leaf.nextIndex, .nextValue = current_leaf.nextValue }); if (!is_already_present) { // Update the current leaf to point it to the new leaf - leaves[current].nextIndex = leaves.size(); - leaves[current].nextValue = value; + current_leaf.nextIndex = leaves.size(); + current_leaf.nextValue = value; + + leaves[current].set(current_leaf); // Insert the new leaf with (nextIndex, nextValue) of the current leaf leaves.push_back(new_leaf); diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp index aa5af2771a..4e8155937a 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp @@ -35,7 +35,7 @@ template class NullifierTree : public MerkleTree { using MerkleTree::zero_hashes_; using MerkleTree::depth_; using MerkleTree::tree_id_; - std::vector leaves; + std::vector leaves; }; extern template class NullifierTree; From 42b7f176974fdaa9e7c249f92e259176b59edd78 Mon Sep 17 00:00:00 2001 From: Suyash Bagad Date: Wed, 19 Apr 2023 17:17:54 +0000 Subject: [PATCH 04/92] More generators for aztec3. --- .../crypto/generators/generator_data.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/src/barretenberg/crypto/generators/generator_data.cpp b/cpp/src/barretenberg/crypto/generators/generator_data.cpp index 1ad6127923..982003c682 100644 --- a/cpp/src/barretenberg/crypto/generators/generator_data.cpp +++ b/cpp/src/barretenberg/crypto/generators/generator_data.cpp @@ -6,14 +6,18 @@ namespace { // The number of unique base points with default main index with precomputed ladders #ifdef __wasm__ -constexpr size_t num_default_generators = 32; +constexpr size_t num_default_generators = 64; +constexpr size_t num_generators_per_hash_index = 16; +constexpr size_t num_hash_indices = 32; +constexpr size_t hash_indices_generator_offset = 64; +// TODO need to resolve memory out of bounds when these are too high #else constexpr size_t num_default_generators = 2048; +constexpr size_t num_hash_indices = 32; +constexpr size_t num_generators_per_hash_index = 128; +constexpr size_t hash_indices_generator_offset = 2048; #endif -constexpr size_t hash_indices_generator_offset = 2048; -constexpr size_t num_hash_indices = 16; -constexpr size_t num_generators_per_hash_index = 8; constexpr size_t num_indexed_generators = num_hash_indices * num_generators_per_hash_index; constexpr size_t size_of_generator_data_array = hash_indices_generator_offset + num_indexed_generators; constexpr size_t num_generator_types = 3; From 16df5ac5627ed12ce2d275f0d03715cb8194c52c Mon Sep 17 00:00:00 2001 From: Suyash Bagad Date: Thu, 20 Apr 2023 09:55:21 +0000 Subject: [PATCH 05/92] update js vk (because we now use UP for merkle hashing) --- .../join_split_example/proofs/join_split/join_split.test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp b/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp index 25b802cd62..10c44f8ec8 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp +++ b/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp @@ -806,7 +806,7 @@ TEST_F(join_split_tests, test_0_input_notes_and_detect_circuit_change) constexpr uint32_t CIRCUIT_GATE_COUNT = 185573; constexpr uint32_t GATES_NEXT_POWER_OF_TWO = 524288; - const uint256_t VK_HASH("13eb88883e80efb9bf306af2962cd1a49e9fa1b0bfb2d4b563b95217a17bcc74"); + const uint256_t VK_HASH("21389d5392ee23ffc96984689150b63d62113678b1ba127346a0ec72df842354"); auto number_of_gates_js = result.number_of_gates; auto vk_hash_js = get_verification_key()->sha256_hash(); From 274c8623f86794f7004ce5eef4e02f855af35eaa Mon Sep 17 00:00:00 2001 From: Suyash Bagad Date: Thu, 20 Apr 2023 15:50:17 +0530 Subject: [PATCH 06/92] Helpers for ECDSA in A3 (#364) * Add `stdlib_keccak` in cmake. Correct an assertion in `to_byte_array` in bigfield. * Add `random_element` to affine element. * negate y conditionally. --- cpp/src/CMakeLists.txt | 3 ++ .../ecc/groups/affine_element.hpp | 7 +++++ .../ecc/groups/affine_element_impl.hpp | 29 +++++++++++++++++++ .../stdlib/primitives/bigfield/bigfield.hpp | 2 +- 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index a90789f9c6..d199134cb6 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -90,6 +90,7 @@ if(WASM) $ $ $ + $ $ $ $ @@ -192,6 +193,7 @@ if(WASM) $ $ $ + $ $ $ $ @@ -227,6 +229,7 @@ else() $ $ $ + $ $ $ $ diff --git a/cpp/src/barretenberg/ecc/groups/affine_element.hpp b/cpp/src/barretenberg/ecc/groups/affine_element.hpp index 81e6099030..293ba2bd53 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element.hpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element.hpp @@ -49,6 +49,13 @@ template class alignas(64) affine_el constexpr bool on_curve() const noexcept; + /** + * @brief Samples a random point on the curve. + * + * @return A randomly chosen point on the curve + */ + static affine_element random_element(numeric::random::Engine* engine = nullptr) noexcept; + /** * @brief Hash a seed value to curve. * diff --git a/cpp/src/barretenberg/ecc/groups/affine_element_impl.hpp b/cpp/src/barretenberg/ecc/groups/affine_element_impl.hpp index 691f03f942..356f2719c4 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element_impl.hpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element_impl.hpp @@ -202,5 +202,34 @@ affine_element affine_element::hash_to_curve(const uint64_ return affine_element(x_out, y_out_); } + +template +affine_element affine_element::random_element(numeric::random::Engine* engine) noexcept +{ + bool found_one = false; + Fq yy; + Fq x; + Fq y; + while (!found_one) { + // Sample a random x-coordinate and check if it satisfies curve equation. + x = Fq::random_element(engine); + yy = x.sqr() * x + T::b; + if constexpr (T::has_a) { + yy += (x * T::a); + } + auto [found_root, y1] = yy.sqrt(); + y = y1; + + // Negate the y-coordinate based on a randomly sampled bit. + bool random_bit = (engine->get_random_uint8() & 1); + if (random_bit) { + y = -y; + } + + found_one = found_root; + } + return affine_element(x, y); +} + } // namespace group_elements } // namespace barretenberg diff --git a/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp b/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp index 002fce1df1..b87bf158c4 100644 --- a/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp +++ b/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp @@ -160,7 +160,7 @@ template class bigfield { field_t lo = binary_basis_limbs[0].element + (binary_basis_limbs[1].element * shift_1); field_t hi = binary_basis_limbs[2].element + (binary_basis_limbs[3].element * shift_1); // n.b. this only works if NUM_LIMB_BITS * 2 is divisible by 8 - ASSERT((NUM_LIMB_BITS / 8) * 8 == NUM_LIMB_BITS); + ASSERT((NUM_LIMB_BITS * 2 / 8) * 8 == NUM_LIMB_BITS * 2); result.write(byte_array(hi, 32 - (NUM_LIMB_BITS / 4))); result.write(byte_array(lo, (NUM_LIMB_BITS / 4))); return result; From 7ed6df42753722ef67bb3e9c61a03dbcd809ebd3 Mon Sep 17 00:00:00 2001 From: Maddiaa <47148561+cheethas@users.noreply.github.com> Date: Tue, 18 Apr 2023 05:12:08 -0700 Subject: [PATCH 07/92] feat(nullifier_tree): make empty nullifier tree leaves hash be 0 (#360) * feat(nullifier_tree): make empty nullifier tree leaves be 0 * fix: add append zero behaviour and test * fix: explicit type name * clean: update class semantics --------- Co-authored-by: cheethas Co-authored-by: cheethas --- .../nullifier_tree/nullifier_leaf.hpp | 69 +++++- .../nullifier_tree/nullifier_memory_tree.cpp | 34 ++- .../nullifier_tree/nullifier_memory_tree.hpp | 10 +- .../nullifier_memory_tree.test.cpp | 216 ++++++++++++++++-- .../nullifier_tree/nullifier_tree.cpp | 23 +- .../nullifier_tree/nullifier_tree.hpp | 2 +- 6 files changed, 304 insertions(+), 50 deletions(-) diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp index bac3478e6a..d7080b48bb 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp @@ -40,14 +40,79 @@ struct nullifier_leaf { barretenberg::fr hash() const { return stdlib::merkle_tree::hash_multiple_native({ value, nextIndex, nextValue }); } }; -inline std::pair find_closest_leaf(std::vector const& leaves_, fr const& new_value) +/** + * @brief Wrapper for the Nullifier leaf class that allows for 0 values + * + */ +class WrappedNullifierLeaf { + + public: + // Initialise with a nullifier leaf + WrappedNullifierLeaf(nullifier_leaf value) + : data(value) + {} + // Initialise an empty leaf + WrappedNullifierLeaf() + : data(std::nullopt) + {} + + bool operator==(WrappedNullifierLeaf const&) const = default; + + /** + * @brief Pass through the underlying std::optional method + * + * @return true + * @return false + */ + bool has_value() const { return data.has_value(); } + + /** + * @brief Return the wrapped nullifier_leaf object + * + * @return nullifier_leaf + */ + nullifier_leaf unwrap() const { return data.value(); } + + /** + * @brief Set the wrapped nullifier_leaf object value + * + * @param value + */ + void set(nullifier_leaf value) { data.emplace(value); } + + /** + * @brief Return the hash of the wrapped object, other return the zero hash of 0 + * + * @return barretenberg::fr + */ + barretenberg::fr hash() const { return data.has_value() ? data.value().hash() : barretenberg::fr::zero(); } + + /** + * @brief Generate a zero leaf (call the constructor with no arguments) + * + * @return NullifierLeaf + */ + static WrappedNullifierLeaf zero() { return WrappedNullifierLeaf(); } + + private: + // Underlying data + std::optional data; +}; + +inline std::pair find_closest_leaf(std::vector const& leaves_, fr const& new_value) { std::vector diff; bool repeated = false; auto new_value_ = uint256_t(new_value); for (size_t i = 0; i < leaves_.size(); i++) { - auto leaf_value_ = uint256_t(leaves_[i].value); + + if (!leaves_[i].has_value()) { + diff.push_back(new_value_); + continue; + } + + auto leaf_value_ = uint256_t(leaves_[i].unwrap().value); if (leaf_value_ > new_value_) { diff.push_back(leaf_value_); } else if (leaf_value_ == new_value_) { diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp index 09398f5a5a..f070ab2ded 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp @@ -12,11 +12,8 @@ NullifierMemoryTree::NullifierMemoryTree(size_t depth) total_size_ = 1UL << depth_; hashes_.resize(total_size_ * 2 - 2); - // Build the entire tree. - nullifier_leaf zero_leaf = { 0, 0, 0 }; - leaves_.push_back(zero_leaf); - auto current = zero_leaf.hash(); - update_element(0, current); + // Build the entire tree and fill with 0 hashes. + auto current = WrappedNullifierLeaf::zero().hash(); size_t layer_size = total_size_; for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { for (size_t i = 0; i < layer_size; ++i) { @@ -25,30 +22,45 @@ NullifierMemoryTree::NullifierMemoryTree(size_t depth) current = hash_pair_native(current, current); } - root_ = current; + // Insert the initial leaf at index 0 + auto initial_leaf = WrappedNullifierLeaf(nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }); + leaves_.push_back(initial_leaf); + root_ = update_element(0, initial_leaf.hash()); } fr NullifierMemoryTree::update_element(fr const& value) { // Find the leaf with the value closest and less than `value` + + // If value is 0 we simply append 0 a null NullifierLeaf to the tree + if (value == 0) { + auto zero_leaf = WrappedNullifierLeaf::zero(); + leaves_.push_back(zero_leaf); + return update_element(leaves_.size() - 1, zero_leaf.hash()); + } + size_t current; bool is_already_present; std::tie(current, is_already_present) = find_closest_leaf(leaves_, value); + nullifier_leaf current_leaf = leaves_[current].unwrap(); nullifier_leaf new_leaf = { .value = value, - .nextIndex = leaves_[current].nextIndex, - .nextValue = leaves_[current].nextValue }; + .nextIndex = current_leaf.nextIndex, + .nextValue = current_leaf.nextValue }; + if (!is_already_present) { // Update the current leaf to point it to the new leaf - leaves_[current].nextIndex = leaves_.size(); - leaves_[current].nextValue = value; + current_leaf.nextIndex = leaves_.size(); + current_leaf.nextValue = value; + + leaves_[current].set(current_leaf); // Insert the new leaf with (nextIndex, nextValue) of the current leaf leaves_.push_back(new_leaf); } // Update the old leaf in the tree - auto old_leaf_hash = leaves_[current].hash(); + auto old_leaf_hash = current_leaf.hash(); size_t old_leaf_index = current; auto root = update_element(old_leaf_index, old_leaf_hash); diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp index 3a41d69194..d40b2ea2bc 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp @@ -68,6 +68,7 @@ using namespace barretenberg; * nextVal 10 50 20 30 0 0 0 0 */ class NullifierMemoryTree : public MemoryTree { + public: NullifierMemoryTree(size_t depth); @@ -78,15 +79,18 @@ class NullifierMemoryTree : public MemoryTree { fr update_element(fr const& value); const std::vector& get_hashes() { return hashes_; } - const std::vector& get_leaves() { return leaves_; } - const nullifier_leaf& get_leaf(size_t index) { return leaves_[index]; } + const WrappedNullifierLeaf get_leaf(size_t index) + { + return (index < leaves_.size()) ? leaves_[index] : WrappedNullifierLeaf::zero(); + } + const std::vector& get_leaves() { return leaves_; } protected: using MemoryTree::depth_; using MemoryTree::hashes_; using MemoryTree::root_; using MemoryTree::total_size_; - std::vector leaves_; + std::vector leaves_; }; } // namespace merkle_tree diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp index b404519fd4..1a5b5bfae8 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp @@ -62,8 +62,8 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(30); EXPECT_EQ(tree.get_leaves().size(), 2); - EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); /** * Add new value 10: @@ -76,9 +76,9 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(10); EXPECT_EQ(tree.get_leaves().size(), 3); - EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 1, 30 }).hash()); /** * Add new value 20: @@ -91,18 +91,18 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), nullifier_leaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); // Adding the same value must not affect anything tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), nullifier_leaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); /** * Add new value 50: @@ -115,11 +115,11 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(50); EXPECT_EQ(tree.get_leaves().size(), 5); - EXPECT_EQ(tree.get_leaves()[0].hash(), nullifier_leaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), nullifier_leaf({ 30, 4, 50 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), nullifier_leaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), nullifier_leaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[4].hash(), nullifier_leaf({ 50, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 4, 50 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf({ 50, 0, 0 }).hash()); // Manually compute the node values auto e000 = tree.get_leaves()[0].hash(); @@ -127,9 +127,9 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) auto e010 = tree.get_leaves()[2].hash(); auto e011 = tree.get_leaves()[3].hash(); auto e100 = tree.get_leaves()[4].hash(); - auto e101 = nullifier_leaf({ 0, 0, 0 }).hash(); - auto e110 = nullifier_leaf({ 0, 0, 0 }).hash(); - auto e111 = nullifier_leaf({ 0, 0, 0 }).hash(); + auto e101 = WrappedNullifierLeaf::zero().hash(); + auto e110 = WrappedNullifierLeaf::zero().hash(); + auto e111 = WrappedNullifierLeaf::zero().hash(); auto e00 = hash_pair_native(e000, e001); auto e01 = hash_pair_native(e010, e011); @@ -161,6 +161,172 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) EXPECT_EQ(tree.get_hash_path(7), expected); } +TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) +{ + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + NullifierMemoryTree tree(depth); + + /** + * Intial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 0 0 0 0 0 0 0 + * nextIdx 0 0 0 0 0 0 0 0 + * nextVal 0 0 0 0 0 0 0 0 + */ + WrappedNullifierLeaf zero_leaf = WrappedNullifierLeaf({ 0, 0, 0 }); + EXPECT_EQ(tree.get_leaves().size(), 1); + EXPECT_EQ(tree.get_leaves()[0], zero_leaf); + + /** + * Add new value 30: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 30 0 0 0 0 0 0 0 + */ + tree.update_element(30); + EXPECT_EQ(tree.get_leaves().size(), 2); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + + /** + * Add new value 10: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 0 0 0 0 0 + * nextIdx 2 0 1 0 0 0 0 0 + * nextVal 10 0 30 0 0 0 0 0 + */ + tree.update_element(10); + EXPECT_EQ(tree.get_leaves().size(), 3); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 1, 30 }).hash()); + + /** + * Add new value 20: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + */ + tree.update_element(20); + EXPECT_EQ(tree.get_leaves().size(), 4); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + + // Adding the same value must not affect anything + tree.update_element(20); + EXPECT_EQ(tree.get_leaves().size(), 4); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + + /** + * Add new value 0: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + */ + tree.update_element(0); + EXPECT_EQ(tree.get_leaves().size(), 5); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf::zero().hash()); + + /* + * Add new value 0: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + */ + tree.update_element(0); + EXPECT_EQ(tree.get_leaves().size(), 6); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves()[5].hash(), WrappedNullifierLeaf::zero().hash()); + + /** + * Add new value 50: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 50 0 + * nextIdx 2 6 3 1 0 0 0 0 + * nextVal 10 50 20 30 0 0 0 0 + */ + tree.update_element(50); + EXPECT_EQ(tree.get_leaves().size(), 7); + EXPECT_EQ(tree.get_leaf(0).hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaf(1).hash(), WrappedNullifierLeaf({ 30, 6, 50 }).hash()); + EXPECT_EQ(tree.get_leaf(2).hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaf(3).hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaf(4).hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaf(5).hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaf(6).hash(), WrappedNullifierLeaf({ 50, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaf(7).hash(), WrappedNullifierLeaf::zero().hash()); + + // Manually compute the node values + auto e000 = tree.get_leaf(0).hash(); + auto e001 = tree.get_leaf(1).hash(); + auto e010 = tree.get_leaf(2).hash(); + auto e011 = tree.get_leaf(3).hash(); + auto e100 = tree.get_leaf(4).hash(); + auto e101 = tree.get_leaf(5).hash(); + auto e110 = tree.get_leaf(6).hash(); + auto e111 = tree.get_leaf(7).hash(); + + auto e00 = hash_pair_native(e000, e001); + auto e01 = hash_pair_native(e010, e011); + auto e10 = hash_pair_native(e100, e101); + auto e11 = hash_pair_native(e110, e111); + + auto e0 = hash_pair_native(e00, e01); + auto e1 = hash_pair_native(e10, e11); + auto root = hash_pair_native(e0, e1); + + // Check the hash path at index 2 and 3 + // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) + fr_hash_path expected = { + std::make_pair(e010, e011), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(2), expected); + EXPECT_EQ(tree.get_hash_path(3), expected); + EXPECT_EQ(tree.root(), root); + + // Check the hash path at index 6 and 7 + expected = { + std::make_pair(e110, e111), + std::make_pair(e10, e11), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(6), expected); + EXPECT_EQ(tree.get_hash_path(7), expected); +} TEST(crypto_nullifier_tree, test_nullifier_tree) { // Create a depth-8 indexed merkle tree @@ -190,8 +356,10 @@ TEST(crypto_nullifier_tree, test_nullifier_tree) const auto& leaves = tree.get_leaves(); std::vector differences; for (size_t i = 0; i < leaves.size(); i++) { - uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(leaves[i].value)); - uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(leaves[i].nextValue)); + uint256_t diff_hi = + abs_diff(uint256_t(new_member), uint256_t(leaves[i].has_value() ? leaves[i].unwrap().value : 0)); + uint256_t diff_lo = + abs_diff(uint256_t(new_member), uint256_t(leaves[i].has_value() ? leaves[i].unwrap().nextValue : 0)); differences.push_back(diff_hi + diff_lo); } auto it = std::min_element(differences.begin(), differences.end()); @@ -199,5 +367,5 @@ TEST(crypto_nullifier_tree, test_nullifier_tree) // Merkle proof at `index` proves non-membership of `new_member` auto hash_path = tree.get_hash_path(index); - EXPECT_TRUE(check_hash_path(tree.root(), hash_path, leaves[index], index)); + EXPECT_TRUE(check_hash_path(tree.root(), hash_path, leaves[index].unwrap(), index)); } \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp index 0a36b8020c..0280670f15 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp @@ -29,10 +29,13 @@ NullifierTree::NullifierTree(Store& store, size_t depth, uint8_t tree_id) // Compute the zero values at each layer. // Insert the zero leaf to the `leaves` and also to the tree at index 0. - auto zero_leaf = nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }; - leaves.push_back(zero_leaf); - auto current = zero_leaf.hash(); - update_element(0, current); + WrappedNullifierLeaf initial_leaf = + WrappedNullifierLeaf(nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }); + leaves.push_back(initial_leaf); + update_element(0, initial_leaf.hash()); + + // Create the zero hashes for the tree + auto current = WrappedNullifierLeaf::zero().hash(); for (size_t i = 0; i < depth; ++i) { zero_hashes_[i] = current; current = hash_pair_native(current, current); @@ -53,13 +56,15 @@ template fr NullifierTree::update_element(fr const& valu bool is_already_present; std::tie(current, is_already_present) = find_closest_leaf(leaves, value); - nullifier_leaf new_leaf = { .value = value, - .nextIndex = leaves[current].nextIndex, - .nextValue = leaves[current].nextValue }; + nullifier_leaf current_leaf = leaves[current].unwrap(); + WrappedNullifierLeaf new_leaf = WrappedNullifierLeaf( + { .value = value, .nextIndex = current_leaf.nextIndex, .nextValue = current_leaf.nextValue }); if (!is_already_present) { // Update the current leaf to point it to the new leaf - leaves[current].nextIndex = leaves.size(); - leaves[current].nextValue = value; + current_leaf.nextIndex = leaves.size(); + current_leaf.nextValue = value; + + leaves[current].set(current_leaf); // Insert the new leaf with (nextIndex, nextValue) of the current leaf leaves.push_back(new_leaf); diff --git a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp index aa5af2771a..4e8155937a 100644 --- a/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp +++ b/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp @@ -35,7 +35,7 @@ template class NullifierTree : public MerkleTree { using MerkleTree::zero_hashes_; using MerkleTree::depth_; using MerkleTree::tree_id_; - std::vector leaves; + std::vector leaves; }; extern template class NullifierTree; From 27f5f8bb9aa41c4099f520e7cb25e448c189657e Mon Sep 17 00:00:00 2001 From: Suyash Bagad Date: Fri, 21 Apr 2023 15:22:23 +0000 Subject: [PATCH 08/92] Change pedersen hash c_bind to use `pedersen_hash::lookup`. --- cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp b/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp index fc28d6fff0..ffffc314dc 100644 --- a/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp +++ b/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp @@ -11,6 +11,7 @@ extern "C" { WASM_EXPORT void pedersen_hash__init() { + // TODO: do we need this if we are using lookup-pedersen in merkle trees? crypto::generators::init_generator_data(); } @@ -18,7 +19,7 @@ WASM_EXPORT void pedersen__hash_pair(uint8_t const* left, uint8_t const* right, { auto lhs = barretenberg::fr::serialize_from_buffer(left); auto rhs = barretenberg::fr::serialize_from_buffer(right); - auto r = crypto::pedersen_hash::hash_multiple({ lhs, rhs }); + auto r = crypto::pedersen_hash::lookup::hash_multiple({ lhs, rhs }); barretenberg::fr::serialize_to_buffer(r, result); } @@ -26,7 +27,7 @@ WASM_EXPORT void pedersen__hash_multiple(uint8_t const* inputs_buffer, uint8_t* { std::vector to_compress; read(inputs_buffer, to_compress); - auto r = crypto::pedersen_hash::hash_multiple(to_compress); + auto r = crypto::pedersen_hash::lookup::hash_multiple(to_compress); barretenberg::fr::serialize_to_buffer(r, output); } @@ -36,7 +37,7 @@ WASM_EXPORT void pedersen__hash_multiple_with_hash_index(uint8_t const* inputs_b { std::vector to_compress; read(inputs_buffer, to_compress); - auto r = crypto::pedersen_hash::hash_multiple(to_compress, hash_index); + auto r = crypto::pedersen_hash::lookup::hash_multiple(to_compress, hash_index); barretenberg::fr::serialize_to_buffer(r, output); } @@ -54,7 +55,7 @@ WASM_EXPORT uint8_t* pedersen__hash_to_tree(uint8_t const* data) fields.reserve(num_outputs); for (size_t i = 0; fields.size() < num_outputs; i += 2) { - fields.push_back(crypto::pedersen_hash::hash_multiple({ fields[i], fields[i + 1] })); + fields.push_back(crypto::pedersen_hash::lookup::hash_multiple({ fields[i], fields[i + 1] })); } auto buf_size = 4 + num_outputs * sizeof(grumpkin::fq); From 2a72649afe534549637009c3fe867d7bc53b45c0 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 18 Apr 2023 17:19:55 -0400 Subject: [PATCH 09/92] feat: add msgpack-c submodule --- .gitmodules | 3 +++ cpp/src/CMakeLists.txt | 3 +++ cpp/src/msgpack-c | 1 + 3 files changed, 7 insertions(+) create mode 160000 cpp/src/msgpack-c diff --git a/.gitmodules b/.gitmodules index 29296995e2..31154308d0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "foundation"] path = foundation url = git@github.com:AztecProtocol/foundation.git +[submodule "cpp/src/msgpack-c"] + path = cpp/src/msgpack-c + url = https://github.com/msgpack/msgpack-c diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index a90789f9c6..00294722fe 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -52,6 +52,9 @@ add_subdirectory(barretenberg/plonk) add_subdirectory(barretenberg/stdlib) add_subdirectory(barretenberg/join_split_example) add_subdirectory(barretenberg/dsl) +add_subdirectory(msgpack-c) +target_compile_options(msgpack-c PRIVATE -Wno-sign-conversion) +target_compile_options(jsonconv PRIVATE -Wno-sign-conversion) if(BENCHMARKS) add_subdirectory(barretenberg/benchmark) diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c new file mode 160000 index 0000000000..b70c6d3a06 --- /dev/null +++ b/cpp/src/msgpack-c @@ -0,0 +1 @@ +Subproject commit b70c6d3a06703f5cca0f3e164362fbb7541a2959 From 1ae2066a7c0f6ead2385c442a7c081ec77f56ea9 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 18 Apr 2023 21:55:16 -0400 Subject: [PATCH 10/92] Give up on msgpack c_master --- barretenberg.code-workspace | 3 ++- cpp/cmake/module.cmake | 6 ++++++ cpp/src/CMakeLists.txt | 3 +++ .../proof_system/polynomial_store/polynomial_store.test.cpp | 2 +- cpp/src/barretenberg/srs/io.test.cpp | 1 + cpp/src/msgpack-c | 2 +- 6 files changed, 14 insertions(+), 3 deletions(-) diff --git a/barretenberg.code-workspace b/barretenberg.code-workspace index c0737c9176..823190927d 100644 --- a/barretenberg.code-workspace +++ b/barretenberg.code-workspace @@ -149,7 +149,8 @@ "cmake.configureArgs": [ "--preset clang15", "-G Ninja", - "-g" + "-g", + "-DMULTITHREADING=OFF" ], "cmake.useCMakePresets": "auto", }, diff --git a/cpp/cmake/module.cmake b/cpp/cmake/module.cmake index 27b94b9adc..910a1becf4 100644 --- a/cpp/cmake/module.cmake +++ b/cpp/cmake/module.cmake @@ -54,6 +54,7 @@ function(barretenberg_module MODULE_NAME) PUBLIC ${ARGN} ${TBB_IMPORTED_TARGETS} + msgpack-c ) list(APPEND lib_targets ${MODULE_NAME}) @@ -74,6 +75,7 @@ function(barretenberg_module MODULE_NAME) PRIVATE GTest::gtest ${TBB_IMPORTED_TARGETS} + msgpack-c ) add_executable( @@ -115,6 +117,7 @@ function(barretenberg_module MODULE_NAME) GTest::gtest GTest::gtest_main ${TBB_IMPORTED_TARGETS} + msgpack-c ) if(NOT WASM AND NOT CI) @@ -184,6 +187,7 @@ function(barretenberg_module MODULE_NAME) ${MODULE_NAME}_${FUZZER_NAME_STEM}_fuzzer PRIVATE ${MODULE_LINK_NAME} + msgpack-c ) endforeach() endif() @@ -202,6 +206,7 @@ function(barretenberg_module MODULE_NAME) PRIVATE benchmark::benchmark ${TBB_IMPORTED_TARGETS} + msgpack-c ) add_executable( @@ -217,6 +222,7 @@ function(barretenberg_module MODULE_NAME) ${ARGN} benchmark::benchmark ${TBB_IMPORTED_TARGETS} + msgpack-c ) add_custom_target( diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 00294722fe..2f87caedba 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -101,6 +101,7 @@ if(WASM) $ $ $ + $ ) # With binaryen installed, it seems its wasm backend optimiser gets invoked automatically. @@ -200,6 +201,7 @@ if(WASM) $ $ $ + $ ) else() # For use when compiling dependent cpp projects @@ -236,6 +238,7 @@ else() $ $ $ + $ ) if(INSTALL_BARRETENBERG) diff --git a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp index a068cce790..38b3138655 100644 --- a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp +++ b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp @@ -102,7 +102,7 @@ TEST(PolynomialStore, RangeBasedFor) polynomial_store.put("id_2", std::move(poly2)); // Check that PolynomialStore meets criteria for std::ranges::range - EXPECT_TRUE(std::ranges::range>); + // EXPECT_TRUE(std::ranges::range>); // For example ... Polynomial polynomial_sum(size); diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index 7ff2806227..164d29aedf 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -3,6 +3,7 @@ #include "io.hpp" #include "barretenberg/common/mem.hpp" #include +#include "msgpack.hpp" using namespace barretenberg; diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index b70c6d3a06..1edfba0cd2 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit b70c6d3a06703f5cca0f3e164362fbb7541a2959 +Subproject commit 1edfba0cd24378bba74578f18ca96e3809ae1664 From db7c2bb1f01bf4b093d020bc2ab22ae42275a1ee Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 19 Apr 2023 00:28:49 -0400 Subject: [PATCH 11/92] Working hacky msgpack test --- cpp/cmake/module.cmake | 6 -- cpp/src/CMakeLists.txt | 9 +- .../crypto/pedersen_commitment/c_bind.cpp | 1 - cpp/src/barretenberg/env/crs.cpp | 10 +-- cpp/src/barretenberg/env/crs.hpp | 6 +- .../plonk/proof_system/types/proof.hpp | 2 +- .../turbo_circuit_constructor.cpp | 2 +- cpp/src/barretenberg/srs/io.test.cpp | 38 ++++++++ cpp/src/barretenberg/srs/msgpack-impl.hpp | 90 +++++++++++++++++++ 9 files changed, 140 insertions(+), 24 deletions(-) create mode 100644 cpp/src/barretenberg/srs/msgpack-impl.hpp diff --git a/cpp/cmake/module.cmake b/cpp/cmake/module.cmake index 910a1becf4..27b94b9adc 100644 --- a/cpp/cmake/module.cmake +++ b/cpp/cmake/module.cmake @@ -54,7 +54,6 @@ function(barretenberg_module MODULE_NAME) PUBLIC ${ARGN} ${TBB_IMPORTED_TARGETS} - msgpack-c ) list(APPEND lib_targets ${MODULE_NAME}) @@ -75,7 +74,6 @@ function(barretenberg_module MODULE_NAME) PRIVATE GTest::gtest ${TBB_IMPORTED_TARGETS} - msgpack-c ) add_executable( @@ -117,7 +115,6 @@ function(barretenberg_module MODULE_NAME) GTest::gtest GTest::gtest_main ${TBB_IMPORTED_TARGETS} - msgpack-c ) if(NOT WASM AND NOT CI) @@ -187,7 +184,6 @@ function(barretenberg_module MODULE_NAME) ${MODULE_NAME}_${FUZZER_NAME_STEM}_fuzzer PRIVATE ${MODULE_LINK_NAME} - msgpack-c ) endforeach() endif() @@ -206,7 +202,6 @@ function(barretenberg_module MODULE_NAME) PRIVATE benchmark::benchmark ${TBB_IMPORTED_TARGETS} - msgpack-c ) add_executable( @@ -222,7 +217,6 @@ function(barretenberg_module MODULE_NAME) ${ARGN} benchmark::benchmark ${TBB_IMPORTED_TARGETS} - msgpack-c ) add_custom_target( diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 2f87caedba..e29bafcd3b 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -27,7 +27,7 @@ if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") add_compile_options(-fconstexpr-ops-limit=100000000) endif() -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/msgpack-c/include) # I feel this should be limited to ecc, however it's currently used in headers that go across libraries, # and there currently isn't an easy way to inherit the DDISABLE_SHENANIGANS parameter. @@ -38,6 +38,7 @@ else() message(STATUS "Using optimized assembly for field arithmetic.") endif() +set(MSGPACK_USE_BOOST OFF CACHE BOOL "Disable boost" FORCE) add_subdirectory(barretenberg/common) add_subdirectory(barretenberg/env) add_subdirectory(barretenberg/numeric) @@ -52,9 +53,6 @@ add_subdirectory(barretenberg/plonk) add_subdirectory(barretenberg/stdlib) add_subdirectory(barretenberg/join_split_example) add_subdirectory(barretenberg/dsl) -add_subdirectory(msgpack-c) -target_compile_options(msgpack-c PRIVATE -Wno-sign-conversion) -target_compile_options(jsonconv PRIVATE -Wno-sign-conversion) if(BENCHMARKS) add_subdirectory(barretenberg/benchmark) @@ -101,7 +99,6 @@ if(WASM) $ $ $ - $ ) # With binaryen installed, it seems its wasm backend optimiser gets invoked automatically. @@ -201,7 +198,6 @@ if(WASM) $ $ $ - $ ) else() # For use when compiling dependent cpp projects @@ -238,7 +234,6 @@ else() $ $ $ - $ ) if(INSTALL_BARRETENBERG) diff --git a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp index 1b58491c3c..f503481ce7 100644 --- a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp +++ b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp @@ -30,7 +30,6 @@ WASM_EXPORT void pedersen_plookup_compress_fields(uint8_t const* left, uint8_t c barretenberg::fr::serialize_to_buffer(r, result); } - WASM_EXPORT void pedersen__compress(uint8_t const* inputs_buffer, uint8_t* output) { std::vector to_compress; diff --git a/cpp/src/barretenberg/env/crs.cpp b/cpp/src/barretenberg/env/crs.cpp index 80561d1c09..923435057d 100644 --- a/cpp/src/barretenberg/env/crs.cpp +++ b/cpp/src/barretenberg/env/crs.cpp @@ -7,13 +7,13 @@ const int NUM_POINTS_IN_TRANSCRIPT = 5040001; - extern "C" { /** * @brief In WASM, loads the verifier reference string. * Used in native code to quickly create an in-memory reference string. */ -uint8_t* env_load_verifier_crs() { +uint8_t* env_load_verifier_crs() +{ std::ifstream transcript; transcript.open("../srs_db/ignition/monomial/transcript00.dat", std::ifstream::binary); // We need two g2 points, each 64 bytes. @@ -33,13 +33,14 @@ uint8_t* env_load_verifier_crs() { * In native code, not intended to be used. * @param num_points The number of points to load. */ -uint8_t* env_load_prover_crs(size_t num_points) { +uint8_t* env_load_prover_crs(size_t num_points) +{ // Note: This implementation is only meant to be instructive. // This should only be used in c-binds to implement the C++ abstractions. std::ifstream transcript; transcript.open("../srs_db/ignition/monomial/transcript00.dat", std::ifstream::binary); // Each g1 point is 64 bytes. - size_t g1_points_size = (num_points) * 64; + size_t g1_points_size = (num_points)*64; std::vector g1_points(g1_points_size); transcript.seekg(28); transcript.read((char*)g1_points.data(), (std::streamsize)g1_points_size); @@ -48,5 +49,4 @@ uint8_t* env_load_prover_crs(size_t num_points) { memcpy(g1_points_copy, g1_points.data(), g1_points_size); return g1_points_copy; } - } \ No newline at end of file diff --git a/cpp/src/barretenberg/env/crs.hpp b/cpp/src/barretenberg/env/crs.hpp index 6a92c92120..871fb817d5 100644 --- a/cpp/src/barretenberg/env/crs.hpp +++ b/cpp/src/barretenberg/env/crs.hpp @@ -2,9 +2,9 @@ #include // To be provided by the environment. -// Outputs from a trusted setup "Common reference string" model: https://en.wikipedia.org/wiki/Common_reference_string_model -// For a WASM build, this is provided by the JavaScript environment. -// For a native build, this is provided in this module. +// Outputs from a trusted setup "Common reference string" model: +// https://en.wikipedia.org/wiki/Common_reference_string_model For a WASM build, this is provided by the JavaScript +// environment. For a native build, this is provided in this module. /** * @brief In WASM, loads the verifier reference string. diff --git a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp index aedf516e04..d0a65aa2b6 100644 --- a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp @@ -28,7 +28,7 @@ template inline void write(B& buf, proof const& data) inline std::ostream& operator<<(std::ostream& os, proof const& data) { // REFACTOR: This is copied from barretenberg/common/streams.hpp, - // which means we could just cout proof_data directly, but that breaks the build in the CI with + // which means we could just cout proof_data directly, but that breaks the build in the CI with // a redefined operator<< error in barretenberg/stdlib/hash/keccak/keccak.test.cpp, // which is something we really don't want to deal with right now. std::ios_base::fmtflags f(os.flags()); diff --git a/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp b/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp index 517587f750..1e2fd973d1 100644 --- a/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp +++ b/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp @@ -1448,7 +1448,7 @@ inline fr TurboCircuitConstructor::fixed_base_gate_evaluation(const size_t gate_ * */ bool TurboCircuitConstructor::check_circuit() { -//#define LAZY_CIRCUIT_CHECKS +// #define LAZY_CIRCUIT_CHECKS #ifdef LAZY_CIRCUIT_CHECKS for (size_t i = 0; i < num_gates; i++) { if (!q_arith[i].is_zero() && !lazy_arithmetic_gate_check(i)) { diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index 164d29aedf..79d7bc6e27 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -4,6 +4,7 @@ #include "barretenberg/common/mem.hpp" #include #include "msgpack.hpp" +#include "msgpack-impl.hpp" using namespace barretenberg; @@ -31,3 +32,40 @@ TEST(io, read_transcript_loads_well_formed_srs) } aligned_free(monomials); } + +struct MyExampleMap { + int a; + std::string b; + + auto serialize_map(const auto& ar) { return ar(NVP(a), NVP(b)); } +}; + +struct MyExampleFlat { + int a; + std::string b; + + auto serialize(const auto& ar) { return ar(a, b); } +}; + +TEST(io, myexample) +{ + + { // pack, unpack + MyExampleMap my{ 42, "Hello" }; + std::stringstream ss; + msgpack::pack(ss, my); + + std::string const& str = ss.str(); + msgpack::object_handle oh = msgpack::unpack(str.data(), str.size()); + msgpack::object obj = oh.get(); + std::cout << obj << std::endl; + // assert(obj.as() == my); + } + { // create object with zone + MyExampleMap my{ 42, "Hello" }; + msgpack::zone z; + msgpack::object obj(my, z); + std::cout << obj << std::endl; + // assert(obj.as() == my); + } +} \ No newline at end of file diff --git a/cpp/src/barretenberg/srs/msgpack-impl.hpp b/cpp/src/barretenberg/srs/msgpack-impl.hpp new file mode 100644 index 0000000000..d93c91219b --- /dev/null +++ b/cpp/src/barretenberg/srs/msgpack-impl.hpp @@ -0,0 +1,90 @@ +#pragma once +// Meant to be included by cbind compilation units. +// Avoid including in headers as it msgpack.hpp includes a good deal of headers. +#include +#include +#include +#include +#include +#define MSGPACK_NO_BOOST +#include + +// A simple name-value pair helper for msgpack serialization +#define NVP(x) #x, x + +// User defined class template specialization +namespace msgpack::adaptor { +struct DefineMapArchive { + template auto operator()(Args&&... args) const + { + return msgpack::type::define_map{ std::forward(args)... }; + } +}; + +struct DefineArchive { + template auto operator()(Args&&... args) const + { + return msgpack::type::define_array{ std::forward(args)... }; + } +}; + +template +concept MapSerializable = requires(T t, DefineMapArchive ar) { + { + t.serialize_map(ar) + } -> std::same_as; + }; +template +concept Serializable = requires(T t, const DefineArchive& ar) { + { + t.serialize(ar) + } -> std::same_as; + }; + +template +// TODO constrain with concept for types that have a msgpack_map() method +struct convert { + msgpack::object const& operator()(msgpack::object const& o, T& v) const + { + return v.serialize_map(DefineMapArchive{}).unpack(o); + } +}; + +template +// TODO constrain with concept for types that have a msgpack() method +struct convert { + msgpack::object const& operator()(msgpack::object const& o, T& v) const + { + return v.serialize(DefineArchive{}).msgpack_unpack(o); + } +}; + +template struct pack { + template packer& operator()(msgpack::packer& o, T const& v) const + { + const_cast(v).serialize_map(DefineMapArchive{}).msgpack_pack(o); + return o; + } +}; + +template struct pack { + template packer& operator()(msgpack::packer& o, T const& v) const + { + const_cast(v).serialize(DefineArchive{}).msgpack_pack(o); + return o; + } +}; + +template struct object_with_zone { + void operator()(msgpack::object::with_zone& o, T const& v) const + { + const_cast(v).serialize_map(DefineMapArchive{}).msgpack_object(&o, o.zone); + } +}; +template struct object_with_zone { + void operator()(msgpack::object::with_zone& o, T const& v) const + { + const_cast(v).serialize(DefineArchive{}).msgpack_object(o); + } +}; +} // namespace msgpack::adaptor From 8a3adc807f1ce344cb9ccce43a0b8b024faa5f9f Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 19 Apr 2023 17:55:40 -0400 Subject: [PATCH 12/92] Interim work --- cpp/src/barretenberg/common/serialize.hpp | 2 + cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp | 2 + .../dsl/acir_format/logic_constraint.hpp | 3 + .../dsl/acir_format/sha256_constraint.hpp | 4 + .../honk/transcript/transcript.hpp | 2 +- cpp/src/barretenberg/srs/io.test.cpp | 132 +++++++++++++++--- cpp/src/barretenberg/srs/msgpack-impl.hpp | 39 +++--- 7 files changed, 142 insertions(+), 42 deletions(-) diff --git a/cpp/src/barretenberg/common/serialize.hpp b/cpp/src/barretenberg/common/serialize.hpp index 622dc919f7..2e18b2b5db 100644 --- a/cpp/src/barretenberg/common/serialize.hpp +++ b/cpp/src/barretenberg/common/serialize.hpp @@ -40,6 +40,8 @@ #ifndef __i386__ __extension__ using uint128_t = unsigned __int128; #endif +// A simple name-value pair helper for msgpack serialization +#define NVP(x) #x, x namespace serialize { // Basic integer read / write, to / from raw buffers. // Pointers to buffers are advanced by length of type. diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp index a094701a5c..5d4904e423 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp @@ -14,6 +14,8 @@ template struct key_pair { struct signature { std::array r; std::array s; + // msgpack entry, update with any new fields + auto serialize(auto ar) { return ar(NVP(r), NVP(s)); } }; template diff --git a/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp index 0a194838ad..5859bf0668 100644 --- a/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp @@ -12,6 +12,9 @@ struct LogicConstraint { uint32_t is_xor_gate; friend bool operator==(LogicConstraint const& lhs, LogicConstraint const& rhs) = default; + + // msgpack entry, update with any new fields + auto serialize(auto ar) { return ar(NVP(a), NVP(b), NVP(result), NVP(num_bits), NVP(is_xor_gate)); } }; void create_logic_gate(plonk::stdlib::types::Composer& composer, diff --git a/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp index dec3bca40a..e6eba969f2 100644 --- a/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp @@ -10,6 +10,8 @@ struct Sha256Input { uint32_t num_bits; friend bool operator==(Sha256Input const& lhs, Sha256Input const& rhs) = default; + // msgpack entry, update with any new fields + auto serialize(auto ar) { return ar(NVP(witness), NVP(num_bits)); } }; struct Sha256Constraint { @@ -17,6 +19,8 @@ struct Sha256Constraint { std::vector result; friend bool operator==(Sha256Constraint const& lhs, Sha256Constraint const& rhs) = default; + // msgpack entry, update with any new fields + auto serialize(auto ar) { return ar(NVP(inputs), NVP(result)); } }; // This function does not work (properly) because the stdlib:sha256 function is not working correctly for 512 bits diff --git a/cpp/src/barretenberg/honk/transcript/transcript.hpp b/cpp/src/barretenberg/honk/transcript/transcript.hpp index ff859806b2..6501003ee5 100644 --- a/cpp/src/barretenberg/honk/transcript/transcript.hpp +++ b/cpp/src/barretenberg/honk/transcript/transcript.hpp @@ -191,7 +191,7 @@ template class ProverTranscript : public BaseTranscript { * @details Serializes the provided object into `proof_data`, and updates the current round state. * * @param label Description/name of the object being added. - * @param element Serializable object that will be added to the transcript + * @param element FlatSerializable object that will be added to the transcript * * @todo Use a concept to only allow certain types to be passed. Requirements are that the object should be * serializable. diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index 79d7bc6e27..1ffc4cb102 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -3,8 +3,10 @@ #include "io.hpp" #include "barretenberg/common/mem.hpp" #include +#include #include "msgpack.hpp" #include "msgpack-impl.hpp" +#include using namespace barretenberg; @@ -33,39 +35,131 @@ TEST(io, read_transcript_loads_well_formed_srs) aligned_free(monomials); } -struct MyExampleMap { +struct MyExampleFlat { int a; std::string b; - auto serialize_map(const auto& ar) { return ar(NVP(a), NVP(b)); } + auto serialize_flat(auto ar) { return ar(a, b); } }; -struct MyExampleFlat { +struct MyExampleMap { int a; std::string b; + MyExampleFlat flat; - auto serialize(const auto& ar) { return ar(a, b); } + auto serialize(auto ar) { return ar(NVP(a), NVP(b), NVP(flat)); } }; +template struct MsgPackSchema; + +struct DefineMapSchema { + auto operator()() { return std::make_tuple(); } + auto operator()(auto& key, const auto& value) + { + (void)value; // unused except for type + // TODO terrible memory leak hack + // since we are serializing proxy objects, these need to exist after this function runs + // but we will only print schemas once then end the program, so hack is livable + return std::make_tuple(std::ref(key), std::ref(*new MsgPackSchema>())); + } + + auto operator()(auto& key, const auto& value, auto&... args) + { + (void)value; // unused except for type + // TODO terrible memory leak hack + // since we are serializing proxy objects, these need to exist after this function runs + // but we will only print schemas once then end the program, so hack is livable + return std::tuple_cat( + std::make_tuple(std::ref(key), std::ref(*new MsgPackSchema>())), + (*this)(args...)); + } +}; + +template const char* schema_name() +{ + const char* result = abi::__cxa_demangle(typeid(T).name(), NULL, NULL, NULL); + std::string result_str = result; + if (result_str.find("basic_string") != std::string::npos) { + return "string"; + } + if (result_str == "i") { + return "int"; + } + return result; +} + +// TODO in hindsight should have been written against just the msgpack_pack api +template struct MsgPackSchema { + const char* type_name_string = "__typename"; + const char* type_name = schema_name(); + void msgpack_pack(auto& packer) const + requires msgpack::adaptor::Serializable + { + msgpack::adaptor::DefineMapArchive ar; + T object; + auto map_definition = object.serialize(ar); + auto archive_wrapper = [&](auto&... args) { return ar(type_name_string, type_name, args...); }; + auto create_map_schema = [&](auto&... args) { + DefineMapSchema schema; + return std::apply(archive_wrapper, schema(args...)); + }; + std::apply(create_map_schema, map_definition.a).msgpack_pack(packer); + } + void msgpack_pack(auto& packer) const + requires msgpack::adaptor::FlatSerializable + { + msgpack::adaptor::DefineArchive ar; + T object; + auto map_definition = object.serialize_flat(ar); + auto archive_wrapper = [&](auto&... args) { return ar(type_name_string, type_name, args...); }; + auto create_map_schema = [&](auto&... args) { + DefineMapSchema schema; + return std::apply(archive_wrapper, schema(args...)); + }; + std::apply(create_map_schema, map_definition.a).msgpack_pack(packer); + } + void msgpack_pack(auto& packer) const + requires(!msgpack::adaptor::Serializable && !msgpack::adaptor::FlatSerializable) + { + packer.pack_str((uint32_t)strlen(type_name)); + packer.pack_str_body(type_name, (uint32_t)strlen(type_name)); + } +}; + +void pretty_print(const auto& obj) +{ + std::stringstream output; + msgpack::pack(output, obj); + std::string output_str = output.str(); + msgpack::object_handle oh = msgpack::unpack(output_str.data(), output_str.size()); + std::cout << "HEY" << std::endl; + std::cout << oh.get() << std::endl; +} + TEST(io, myexample) { { // pack, unpack - MyExampleMap my{ 42, "Hello" }; - std::stringstream ss; - msgpack::pack(ss, my); - - std::string const& str = ss.str(); - msgpack::object_handle oh = msgpack::unpack(str.data(), str.size()); - msgpack::object obj = oh.get(); - std::cout << obj << std::endl; - // assert(obj.as() == my); - } - { // create object with zone - MyExampleMap my{ 42, "Hello" }; - msgpack::zone z; - msgpack::object obj(my, z); - std::cout << obj << std::endl; + MyExampleMap my{ 1, "2", { 3, "4" } }; + MsgPackSchema my_schema; + pretty_print(my); + pretty_print(my_schema); + // std::stringstream ss; + // msgpack::pack(ss, my); + // + // std::string const& str = ss.str(); + // // Write the packed data to a file + // std::ofstream ofs("output.msgpack", std::ios::binary); + // if (ofs) { + // ofs.write(str.data(), (std::streamsize)str.size()); + // ofs.close(); + // std::cout << "Binary string written to output.msgpack" << std::endl; + // } else { + // std::cerr << "Error: Unable to open output.msgpack" << std::endl; + // } + // msgpack::object_handle oh = msgpack::unpack(str.data(), str.size()); + // msgpack::object obj = oh.get(); + // std::cout << obj << std::endl; // assert(obj.as() == my); } } \ No newline at end of file diff --git a/cpp/src/barretenberg/srs/msgpack-impl.hpp b/cpp/src/barretenberg/srs/msgpack-impl.hpp index d93c91219b..e9f66b01e8 100644 --- a/cpp/src/barretenberg/srs/msgpack-impl.hpp +++ b/cpp/src/barretenberg/srs/msgpack-impl.hpp @@ -14,6 +14,9 @@ // User defined class template specialization namespace msgpack::adaptor { +template +concept AnyType = true; + struct DefineMapArchive { template auto operator()(Args&&... args) const { @@ -29,62 +32,54 @@ struct DefineArchive { }; template -concept MapSerializable = requires(T t, DefineMapArchive ar) { - { - t.serialize_map(ar) - } -> std::same_as; - }; +concept Serializable = requires(T t, DefineMapArchive ar) { t.serialize(ar); }; template -concept Serializable = requires(T t, const DefineArchive& ar) { - { - t.serialize(ar) - } -> std::same_as; - }; +concept FlatSerializable = requires(T t, const DefineArchive& ar) { t.serialize_flat(ar); }; -template +template // TODO constrain with concept for types that have a msgpack_map() method struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const { - return v.serialize_map(DefineMapArchive{}).unpack(o); + return v.serialize(DefineMapArchive{}).unpack(o); } }; -template +template // TODO constrain with concept for types that have a msgpack() method struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const { - return v.serialize(DefineArchive{}).msgpack_unpack(o); + return v.serialize_flat(DefineArchive{}).msgpack_unpack(o); } }; -template struct pack { +template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { - const_cast(v).serialize_map(DefineMapArchive{}).msgpack_pack(o); + const_cast(v).serialize(DefineMapArchive{}).msgpack_pack(o); return o; } }; -template struct pack { +template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { - const_cast(v).serialize(DefineArchive{}).msgpack_pack(o); + const_cast(v).serialize_flat(DefineArchive{}).msgpack_pack(o); return o; } }; -template struct object_with_zone { +template struct object_with_zone { void operator()(msgpack::object::with_zone& o, T const& v) const { - const_cast(v).serialize_map(DefineMapArchive{}).msgpack_object(&o, o.zone); + const_cast(v).serialize(DefineMapArchive{}).msgpack_object(&o, o.zone); } }; -template struct object_with_zone { +template struct object_with_zone { void operator()(msgpack::object::with_zone& o, T const& v) const { - const_cast(v).serialize(DefineArchive{}).msgpack_object(o); + const_cast(v).serialize_flat(DefineArchive{}).msgpack_object(o); } }; } // namespace msgpack::adaptor From b475c3f448f72932e712a8c88303251b9e8d5e09 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 19 Apr 2023 17:57:34 -0400 Subject: [PATCH 13/92] Interim work --- cpp/src/barretenberg/srs/io.test.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index 1ffc4cb102..fe644792d1 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -110,13 +110,14 @@ template struct MsgPackSchema { { msgpack::adaptor::DefineArchive ar; T object; - auto map_definition = object.serialize_flat(ar); - auto archive_wrapper = [&](auto&... args) { return ar(type_name_string, type_name, args...); }; - auto create_map_schema = [&](auto&... args) { - DefineMapSchema schema; - return std::apply(archive_wrapper, schema(args...)); + auto array_definition = object.serialize_flat(ar); + auto create_array_schema = [&](auto&... args) { + // TODO terrible memory leak hack + // but we will only print schemas once then end the program + // since we are serializing proxy objects, these need to exist after this function runs + return ar(type_name, *new MsgPackSchema>{}...); }; - std::apply(create_map_schema, map_definition.a).msgpack_pack(packer); + std::apply(create_array_schema, array_definition.a).msgpack_pack(packer); } void msgpack_pack(auto& packer) const requires(!msgpack::adaptor::Serializable && !msgpack::adaptor::FlatSerializable) @@ -132,7 +133,6 @@ void pretty_print(const auto& obj) msgpack::pack(output, obj); std::string output_str = output.str(); msgpack::object_handle oh = msgpack::unpack(output_str.data(), output_str.size()); - std::cout << "HEY" << std::endl; std::cout << oh.get() << std::endl; } From 5769a0cd984c2aaace9a6bcf65752137c4d4a5ea Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 19 Apr 2023 18:01:25 -0400 Subject: [PATCH 14/92] Getting rid of memory hacks --- cpp/src/barretenberg/srs/io.test.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index fe644792d1..d849573a5b 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -111,13 +111,12 @@ template struct MsgPackSchema { msgpack::adaptor::DefineArchive ar; T object; auto array_definition = object.serialize_flat(ar); - auto create_array_schema = [&](auto&... args) { - // TODO terrible memory leak hack - // but we will only print schemas once then end the program - // since we are serializing proxy objects, these need to exist after this function runs - return ar(type_name, *new MsgPackSchema>{}...); + auto archive_wrapper = [&](auto&... args) { return ar(type_name, args...); }; + auto pack_array_schema = [&](auto&... args) { + auto schema = std::make_tuple(MsgPackSchema>{}...); + std::apply(archive_wrapper, schema).msgpack_pack(packer); }; - std::apply(create_array_schema, array_definition.a).msgpack_pack(packer); + std::apply(pack_array_schema, array_definition.a); } void msgpack_pack(auto& packer) const requires(!msgpack::adaptor::Serializable && !msgpack::adaptor::FlatSerializable) From 22a2c5d0a05b9f512f990b3830abeec50bab7b8c Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 19 Apr 2023 18:04:42 -0400 Subject: [PATCH 15/92] fix: memory leaks --- cpp/src/barretenberg/srs/io.test.cpp | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index d849573a5b..8558291a1d 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -54,24 +54,16 @@ template struct MsgPackSchema; struct DefineMapSchema { auto operator()() { return std::make_tuple(); } - auto operator()(auto& key, const auto& value) + auto operator()(auto key, auto value) { (void)value; // unused except for type - // TODO terrible memory leak hack - // since we are serializing proxy objects, these need to exist after this function runs - // but we will only print schemas once then end the program, so hack is livable - return std::make_tuple(std::ref(key), std::ref(*new MsgPackSchema>())); + return std::make_tuple(key, MsgPackSchema()); } - auto operator()(auto& key, const auto& value, auto&... args) + auto operator()(auto key, auto value, auto&... args) { (void)value; // unused except for type - // TODO terrible memory leak hack - // since we are serializing proxy objects, these need to exist after this function runs - // but we will only print schemas once then end the program, so hack is livable - return std::tuple_cat( - std::make_tuple(std::ref(key), std::ref(*new MsgPackSchema>())), - (*this)(args...)); + return std::tuple_cat(std::make_tuple(key, MsgPackSchema()), (*this)(args...)); } }; @@ -88,7 +80,6 @@ template const char* schema_name() return result; } -// TODO in hindsight should have been written against just the msgpack_pack api template struct MsgPackSchema { const char* type_name_string = "__typename"; const char* type_name = schema_name(); @@ -99,11 +90,11 @@ template struct MsgPackSchema { T object; auto map_definition = object.serialize(ar); auto archive_wrapper = [&](auto&... args) { return ar(type_name_string, type_name, args...); }; - auto create_map_schema = [&](auto&... args) { - DefineMapSchema schema; - return std::apply(archive_wrapper, schema(args...)); + auto pack_map_schema = [&](auto&... args) { + auto schema = DefineMapSchema{}(args...); + std::apply(archive_wrapper, schema).msgpack_pack(packer); }; - std::apply(create_map_schema, map_definition.a).msgpack_pack(packer); + std::apply(pack_map_schema, map_definition.a); } void msgpack_pack(auto& packer) const requires msgpack::adaptor::FlatSerializable From e110c271102bd944bc672898c706683ee8792a30 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 19 Apr 2023 18:45:02 -0400 Subject: [PATCH 16/92] Start of demoing cbinds --- cpp/src/barretenberg/srs/io.test.cpp | 12 ++++++++++++ cpp/src/barretenberg/srs/msgpack-impl.hpp | 16 ++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index 8558291a1d..f5892095e1 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -117,6 +117,18 @@ template struct MsgPackSchema { } }; +struct aes__decrypt_buffer_cbc { + std::vector in; + std::vector iv; + std::vector key; + size_t length; + void operator()() {} + auto serialize_flat() {} +}; + +auto aes__decrypt_buffer_cbc(uint8_t* in, uint8_t* iv, const uint8_t* key, const size_t length, uint8_t* r) {} +auto cbind_example() {} + void pretty_print(const auto& obj) { std::stringstream output; diff --git a/cpp/src/barretenberg/srs/msgpack-impl.hpp b/cpp/src/barretenberg/srs/msgpack-impl.hpp index e9f66b01e8..f859995c42 100644 --- a/cpp/src/barretenberg/srs/msgpack-impl.hpp +++ b/cpp/src/barretenberg/srs/msgpack-impl.hpp @@ -32,16 +32,16 @@ struct DefineArchive { }; template -concept Serializable = requires(T t, DefineMapArchive ar) { t.serialize(ar); }; +concept Serializable = requires(T t, DefineMapArchive ar) { t.msgpack(ar); }; template -concept FlatSerializable = requires(T t, const DefineArchive& ar) { t.serialize_flat(ar); }; +concept FlatSerializable = requires(T t, const DefineArchive& ar) { t.nsgpack_flat(ar); }; template // TODO constrain with concept for types that have a msgpack_map() method struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const { - return v.serialize(DefineMapArchive{}).unpack(o); + return v.msgpack(DefineMapArchive{}).unpack(o); } }; @@ -50,14 +50,14 @@ template struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const { - return v.serialize_flat(DefineArchive{}).msgpack_unpack(o); + return v.msgpack_flat(DefineArchive{}).msgpack_unpack(o); } }; template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { - const_cast(v).serialize(DefineMapArchive{}).msgpack_pack(o); + const_cast(v).msgpack(DefineMapArchive{}).msgpack_pack(o); return o; } }; @@ -65,7 +65,7 @@ template struct pack { template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { - const_cast(v).serialize_flat(DefineArchive{}).msgpack_pack(o); + const_cast(v).msgpack_flat(DefineArchive{}).msgpack_pack(o); return o; } }; @@ -73,13 +73,13 @@ template struct pack { template struct object_with_zone { void operator()(msgpack::object::with_zone& o, T const& v) const { - const_cast(v).serialize(DefineMapArchive{}).msgpack_object(&o, o.zone); + const_cast(v).msgpack(DefineMapArchive{}).msgpack_object(&o, o.zone); } }; template struct object_with_zone { void operator()(msgpack::object::with_zone& o, T const& v) const { - const_cast(v).serialize_flat(DefineArchive{}).msgpack_object(o); + const_cast(v).msgpack_flat(DefineArchive{}).msgpack_object(o); } }; } // namespace msgpack::adaptor From 0077d20d0bf666d33a1159996136be7e0e88c0ef Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 19 Apr 2023 18:50:30 -0400 Subject: [PATCH 17/92] Align with other methods --- cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp | 2 +- .../dsl/acir_format/logic_constraint.hpp | 2 +- .../dsl/acir_format/sha256_constraint.hpp | 4 +-- .../honk/transcript/transcript.hpp | 2 +- cpp/src/barretenberg/srs/io.test.cpp | 36 +++++++++---------- cpp/src/barretenberg/srs/msgpack-impl.hpp | 16 ++++----- 6 files changed, 31 insertions(+), 31 deletions(-) diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp index 5d4904e423..3ca0d3c3bd 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp @@ -15,7 +15,7 @@ struct signature { std::array r; std::array s; // msgpack entry, update with any new fields - auto serialize(auto ar) { return ar(NVP(r), NVP(s)); } + auto msgpack(auto ar) { return ar(NVP(r), NVP(s)); } }; template diff --git a/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp index 5859bf0668..b690e4e034 100644 --- a/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp @@ -14,7 +14,7 @@ struct LogicConstraint { friend bool operator==(LogicConstraint const& lhs, LogicConstraint const& rhs) = default; // msgpack entry, update with any new fields - auto serialize(auto ar) { return ar(NVP(a), NVP(b), NVP(result), NVP(num_bits), NVP(is_xor_gate)); } + auto msgpack(auto ar) { return ar(NVP(a), NVP(b), NVP(result), NVP(num_bits), NVP(is_xor_gate)); } }; void create_logic_gate(plonk::stdlib::types::Composer& composer, diff --git a/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp index e6eba969f2..a6a5793cba 100644 --- a/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp @@ -11,7 +11,7 @@ struct Sha256Input { friend bool operator==(Sha256Input const& lhs, Sha256Input const& rhs) = default; // msgpack entry, update with any new fields - auto serialize(auto ar) { return ar(NVP(witness), NVP(num_bits)); } + auto msgpack(auto ar) { return ar(NVP(witness), NVP(num_bits)); } }; struct Sha256Constraint { @@ -20,7 +20,7 @@ struct Sha256Constraint { friend bool operator==(Sha256Constraint const& lhs, Sha256Constraint const& rhs) = default; // msgpack entry, update with any new fields - auto serialize(auto ar) { return ar(NVP(inputs), NVP(result)); } + auto msgpack(auto ar) { return ar(NVP(inputs), NVP(result)); } }; // This function does not work (properly) because the stdlib:sha256 function is not working correctly for 512 bits diff --git a/cpp/src/barretenberg/honk/transcript/transcript.hpp b/cpp/src/barretenberg/honk/transcript/transcript.hpp index 6501003ee5..86cd4a1ee2 100644 --- a/cpp/src/barretenberg/honk/transcript/transcript.hpp +++ b/cpp/src/barretenberg/honk/transcript/transcript.hpp @@ -191,7 +191,7 @@ template class ProverTranscript : public BaseTranscript { * @details Serializes the provided object into `proof_data`, and updates the current round state. * * @param label Description/name of the object being added. - * @param element FlatSerializable object that will be added to the transcript + * @param element HasMsgPackFlat object that will be added to the transcript * * @todo Use a concept to only allow certain types to be passed. Requirements are that the object should be * serializable. diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index f5892095e1..9c40f55964 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -39,7 +39,7 @@ struct MyExampleFlat { int a; std::string b; - auto serialize_flat(auto ar) { return ar(a, b); } + auto msgpack_flat(auto ar) { return ar(a, b); } }; struct MyExampleMap { @@ -47,7 +47,7 @@ struct MyExampleMap { std::string b; MyExampleFlat flat; - auto serialize(auto ar) { return ar(NVP(a), NVP(b), NVP(flat)); } + auto msgpack(auto ar) { return ar(NVP(a), NVP(b), NVP(flat)); } }; template struct MsgPackSchema; @@ -84,11 +84,11 @@ template struct MsgPackSchema { const char* type_name_string = "__typename"; const char* type_name = schema_name(); void msgpack_pack(auto& packer) const - requires msgpack::adaptor::Serializable + requires msgpack::adaptor::HasMsgPack { msgpack::adaptor::DefineMapArchive ar; T object; - auto map_definition = object.serialize(ar); + auto map_definition = object.msgpack(ar); auto archive_wrapper = [&](auto&... args) { return ar(type_name_string, type_name, args...); }; auto pack_map_schema = [&](auto&... args) { auto schema = DefineMapSchema{}(args...); @@ -97,11 +97,11 @@ template struct MsgPackSchema { std::apply(pack_map_schema, map_definition.a); } void msgpack_pack(auto& packer) const - requires msgpack::adaptor::FlatSerializable + requires msgpack::adaptor::HasMsgPackFlat { msgpack::adaptor::DefineArchive ar; T object; - auto array_definition = object.serialize_flat(ar); + auto array_definition = object.msgpack_flat(ar); auto archive_wrapper = [&](auto&... args) { return ar(type_name, args...); }; auto pack_array_schema = [&](auto&... args) { auto schema = std::make_tuple(MsgPackSchema>{}...); @@ -110,24 +110,24 @@ template struct MsgPackSchema { std::apply(pack_array_schema, array_definition.a); } void msgpack_pack(auto& packer) const - requires(!msgpack::adaptor::Serializable && !msgpack::adaptor::FlatSerializable) + requires(!msgpack::adaptor::HasMsgPack && !msgpack::adaptor::HasMsgPackFlat) { packer.pack_str((uint32_t)strlen(type_name)); packer.pack_str_body(type_name, (uint32_t)strlen(type_name)); } }; -struct aes__decrypt_buffer_cbc { - std::vector in; - std::vector iv; - std::vector key; - size_t length; - void operator()() {} - auto serialize_flat() {} -}; - -auto aes__decrypt_buffer_cbc(uint8_t* in, uint8_t* iv, const uint8_t* key, const size_t length, uint8_t* r) {} -auto cbind_example() {} +// struct aes__decrypt_buffer_cbc { +// std::vector in; +// std::vector iv; +// std::vector key; +// size_t length; +// void operator()() {} +// auto msgpack_flat() {} +// }; + +// auto aes__decrypt_buffer_cbc(uint8_t* in, uint8_t* iv, const uint8_t* key, const size_t length, uint8_t* r) {} +// auto cbind_example() {} void pretty_print(const auto& obj) { diff --git a/cpp/src/barretenberg/srs/msgpack-impl.hpp b/cpp/src/barretenberg/srs/msgpack-impl.hpp index f859995c42..d494d198e8 100644 --- a/cpp/src/barretenberg/srs/msgpack-impl.hpp +++ b/cpp/src/barretenberg/srs/msgpack-impl.hpp @@ -32,11 +32,11 @@ struct DefineArchive { }; template -concept Serializable = requires(T t, DefineMapArchive ar) { t.msgpack(ar); }; +concept HasMsgPack = requires(T t, DefineMapArchive ar) { t.msgpack(ar); }; template -concept FlatSerializable = requires(T t, const DefineArchive& ar) { t.nsgpack_flat(ar); }; +concept HasMsgPackFlat = requires(T t, const DefineArchive& ar) { t.msgpack_flat(ar); }; -template +template // TODO constrain with concept for types that have a msgpack_map() method struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const @@ -45,7 +45,7 @@ struct convert { } }; -template +template // TODO constrain with concept for types that have a msgpack() method struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const @@ -54,7 +54,7 @@ struct convert { } }; -template struct pack { +template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { const_cast(v).msgpack(DefineMapArchive{}).msgpack_pack(o); @@ -62,7 +62,7 @@ template struct pack { } }; -template struct pack { +template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { const_cast(v).msgpack_flat(DefineArchive{}).msgpack_pack(o); @@ -70,13 +70,13 @@ template struct pack { } }; -template struct object_with_zone { +template struct object_with_zone { void operator()(msgpack::object::with_zone& o, T const& v) const { const_cast(v).msgpack(DefineMapArchive{}).msgpack_object(&o, o.zone); } }; -template struct object_with_zone { +template struct object_with_zone { void operator()(msgpack::object::with_zone& o, T const& v) const { const_cast(v).msgpack_flat(DefineArchive{}).msgpack_object(o); From a81f909466d01d548384342744cb246bd81cf157 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 19 Apr 2023 19:10:42 -0400 Subject: [PATCH 18/92] chore: Remove need to return from msgpack method --- cpp/src/barretenberg/srs/io.test.cpp | 89 ++++++++++++----------- cpp/src/barretenberg/srs/msgpack-impl.hpp | 42 +++++------ 2 files changed, 66 insertions(+), 65 deletions(-) diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index 9c40f55964..2ced6bfbf4 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -39,7 +39,7 @@ struct MyExampleFlat { int a; std::string b; - auto msgpack_flat(auto ar) { return ar(a, b); } + void msgpack_flat(auto ar) { ar(a, b); } }; struct MyExampleMap { @@ -47,7 +47,7 @@ struct MyExampleMap { std::string b; MyExampleFlat flat; - auto msgpack(auto ar) { return ar(NVP(a), NVP(b), NVP(flat)); } + void msgpack(auto ar) { ar(NVP(a), NVP(b), NVP(flat)); } }; template struct MsgPackSchema; @@ -86,28 +86,30 @@ template struct MsgPackSchema { void msgpack_pack(auto& packer) const requires msgpack::adaptor::HasMsgPack { - msgpack::adaptor::DefineMapArchive ar; T object; - auto map_definition = object.msgpack(ar); - auto archive_wrapper = [&](auto&... args) { return ar(type_name_string, type_name, args...); }; - auto pack_map_schema = [&](auto&... args) { - auto schema = DefineMapSchema{}(args...); - std::apply(archive_wrapper, schema).msgpack_pack(packer); - }; - std::apply(pack_map_schema, map_definition.a); + object.msgpack([&](auto&... args) { + msgpack::adaptor::DefineMapArchive ar; + auto archive_wrapper = [&](auto&... args) { return ar(type_name_string, type_name, args...); }; + auto pack_map_schema = [&](auto&... args) { + auto schema = DefineMapSchema{}(args...); + std::apply(archive_wrapper, schema).msgpack_pack(packer); + }; + std::apply(pack_map_schema, ar(args...).a); + }); } void msgpack_pack(auto& packer) const requires msgpack::adaptor::HasMsgPackFlat { - msgpack::adaptor::DefineArchive ar; T object; - auto array_definition = object.msgpack_flat(ar); - auto archive_wrapper = [&](auto&... args) { return ar(type_name, args...); }; - auto pack_array_schema = [&](auto&... args) { - auto schema = std::make_tuple(MsgPackSchema>{}...); - std::apply(archive_wrapper, schema).msgpack_pack(packer); - }; - std::apply(pack_array_schema, array_definition.a); + object.msgpack_flat([&](auto&... args) { + msgpack::adaptor::DefineArchive ar; + auto archive_wrapper = [&](auto&... args) { return ar(type_name, args...); }; + auto pack_array_schema = [&](auto&... args) { + auto schema = std::make_tuple(MsgPackSchema>{}...); + std::apply(archive_wrapper, schema).msgpack_pack(packer); + }; + std::apply(pack_array_schema, ar(args...).a); + }); } void msgpack_pack(auto& packer) const requires(!msgpack::adaptor::HasMsgPack && !msgpack::adaptor::HasMsgPackFlat) @@ -117,14 +119,14 @@ template struct MsgPackSchema { } }; -// struct aes__decrypt_buffer_cbc { -// std::vector in; -// std::vector iv; -// std::vector key; -// size_t length; -// void operator()() {} -// auto msgpack_flat() {} -// }; +struct aes__decrypt_buffer_cbc { + std::vector in; + std::vector iv; + std::vector key; + size_t length; + void operator()() {} + auto msgpack_flat(auto ar) { return ar(in, iv, key, length); } +}; // auto aes__decrypt_buffer_cbc(uint8_t* in, uint8_t* iv, const uint8_t* key, const size_t length, uint8_t* r) {} // auto cbind_example() {} @@ -146,22 +148,23 @@ TEST(io, myexample) MsgPackSchema my_schema; pretty_print(my); pretty_print(my_schema); - // std::stringstream ss; - // msgpack::pack(ss, my); - // - // std::string const& str = ss.str(); - // // Write the packed data to a file - // std::ofstream ofs("output.msgpack", std::ios::binary); - // if (ofs) { - // ofs.write(str.data(), (std::streamsize)str.size()); - // ofs.close(); - // std::cout << "Binary string written to output.msgpack" << std::endl; - // } else { - // std::cerr << "Error: Unable to open output.msgpack" << std::endl; - // } - // msgpack::object_handle oh = msgpack::unpack(str.data(), str.size()); - // msgpack::object obj = oh.get(); - // std::cout << obj << std::endl; - // assert(obj.as() == my); + std::stringstream ss; + msgpack::pack(ss, my); + + std::string const& str = ss.str(); + // Write the packed data to a file + std::ofstream ofs("output.msgpack", std::ios::binary); + if (ofs) { + ofs.write(str.data(), (std::streamsize)str.size()); + ofs.close(); + std::cout << "Binary string written to output.msgpack" << std::endl; + } else { + std::cerr << "Error: Unable to open output.msgpack" << std::endl; + } + msgpack::object_handle oh = msgpack::unpack(str.data(), str.size()); + msgpack::object obj = oh.get(); + std::cout << obj << std::endl; + MyExampleMap map = obj.convert(); + std::cout << map.b << std::endl; } } \ No newline at end of file diff --git a/cpp/src/barretenberg/srs/msgpack-impl.hpp b/cpp/src/barretenberg/srs/msgpack-impl.hpp index d494d198e8..a3f6efb66e 100644 --- a/cpp/src/barretenberg/srs/msgpack-impl.hpp +++ b/cpp/src/barretenberg/srs/msgpack-impl.hpp @@ -36,28 +36,26 @@ concept HasMsgPack = requires(T t, DefineMapArchive ar) { t.msgpack(ar); }; template concept HasMsgPackFlat = requires(T t, const DefineArchive& ar) { t.msgpack_flat(ar); }; -template -// TODO constrain with concept for types that have a msgpack_map() method -struct convert { +template struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const { - return v.msgpack(DefineMapArchive{}).unpack(o); + v.msgpack([&](auto&... args) { DefineMapArchive{}(args...).msgpack_unpack(o); }); + return o; } }; -template -// TODO constrain with concept for types that have a msgpack() method -struct convert { +template struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const { - return v.msgpack_flat(DefineArchive{}).msgpack_unpack(o); + v.msgpack_flat([&](auto&... args) { DefineArchive{}(args...).msgpack_unpack(o); }); + return o; } }; template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { - const_cast(v).msgpack(DefineMapArchive{}).msgpack_pack(o); + const_cast(v).msgpack([&](auto&... args) { DefineMapArchive{}(args...).msgpack_pack(o); }); return o; } }; @@ -65,21 +63,21 @@ template struct pack { template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { - const_cast(v).msgpack_flat(DefineArchive{}).msgpack_pack(o); + const_cast(v).msgpack_flat([&](auto&... args) { DefineArchive{}(args...).msgpack_pack(o); }); return o; } }; -template struct object_with_zone { - void operator()(msgpack::object::with_zone& o, T const& v) const - { - const_cast(v).msgpack(DefineMapArchive{}).msgpack_object(&o, o.zone); - } -}; -template struct object_with_zone { - void operator()(msgpack::object::with_zone& o, T const& v) const - { - const_cast(v).msgpack_flat(DefineArchive{}).msgpack_object(o); - } -}; +// template struct object_with_zone { +// void operator()(msgpack::object::with_zone& o, T const& v) const +// { +// const_cast(v).msgpack(DefineMapArchive{}).msgpack_object(&o, o.zone); +// } +// }; +// template struct object_with_zone { +// void operator()(msgpack::object::with_zone& o, T const& v) const +// { +// const_cast(v).msgpack_flat(DefineArchive{}).msgpack_object(o); +// } +// }; } // namespace msgpack::adaptor From 4abc2212d937bb8ff975ae1398bfce5d08404bf0 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 19 Apr 2023 20:00:31 -0400 Subject: [PATCH 19/92] Iterate example --- cpp/src/barretenberg/srs/io.test.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index 2ced6bfbf4..937c758648 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -6,6 +6,7 @@ #include #include "msgpack.hpp" #include "msgpack-impl.hpp" +#include "barretenberg/crypto/aes128/aes128.hpp" #include using namespace barretenberg; @@ -81,6 +82,7 @@ template const char* schema_name() } template struct MsgPackSchema { + // A viable reference is needed for define_map const char* type_name_string = "__typename"; const char* type_name = schema_name(); void msgpack_pack(auto& packer) const @@ -119,14 +121,20 @@ template struct MsgPackSchema { } }; +namespace cbinds { struct aes__decrypt_buffer_cbc { std::vector in; std::vector iv; std::vector key; size_t length; - void operator()() {} - auto msgpack_flat(auto ar) { return ar(in, iv, key, length); } + void msgpack_flat(auto ar) { ar(in, iv, key, length); } + auto operator()() + { + crypto::aes128::decrypt_buffer_cbc(in.data(), iv.data(), key.data(), length); + return in; + } }; +} // namespace cbinds // auto aes__decrypt_buffer_cbc(uint8_t* in, uint8_t* iv, const uint8_t* key, const size_t length, uint8_t* r) {} // auto cbind_example() {} From 8f4fb51923e270310d0eb0872f1876ff4ca880eb Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 20 Apr 2023 09:17:56 -0400 Subject: [PATCH 20/92] fix: Hack around generator issues --- cpp/src/barretenberg/crypto/generators/generator_data.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/barretenberg/crypto/generators/generator_data.cpp b/cpp/src/barretenberg/crypto/generators/generator_data.cpp index 3c07a4348a..bdb9a76479 100644 --- a/cpp/src/barretenberg/crypto/generators/generator_data.cpp +++ b/cpp/src/barretenberg/crypto/generators/generator_data.cpp @@ -17,7 +17,6 @@ constexpr size_t num_generators_per_hash_index = 128; #endif constexpr size_t hash_indices_generator_offset = 2048; - constexpr size_t num_indexed_generators = num_hash_indices * num_generators_per_hash_index; constexpr size_t size_of_generator_data_array = hash_indices_generator_offset + num_indexed_generators; constexpr size_t num_generator_types = 3; From d4901fbb22aba30aad25c57318ca6feb0a4d3370 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 08:39:09 -0400 Subject: [PATCH 21/92] feat: iterate on msgpack in bb --- cpp/CMakeLists.txt | 1 + cpp/src/barretenberg/common/msgpack-cbind.hpp | 34 ++++++ .../{srs => common}/msgpack-impl.hpp | 24 ++-- .../barretenberg/common/msgpack-schema.hpp | 105 ++++++++++++++++++ cpp/src/barretenberg/ecc/fields/field.hpp | 5 +- .../barretenberg/ecc/fields/field_impl.hpp | 24 ++++ cpp/src/barretenberg/ecc/fields/msgpack.cpp | 20 ++++ cpp/src/barretenberg/srs/io.test.cpp | 71 ------------ .../stdlib/primitives/address/address.hpp | 3 + 9 files changed, 201 insertions(+), 86 deletions(-) create mode 100644 cpp/src/barretenberg/common/msgpack-cbind.hpp rename cpp/src/barretenberg/{srs => common}/msgpack-impl.hpp (77%) create mode 100644 cpp/src/barretenberg/common/msgpack-schema.hpp create mode 100644 cpp/src/barretenberg/ecc/fields/msgpack.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 582d728537..0177dcf954 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -74,6 +74,7 @@ if(FUZZING) set(MULTITHREADING OFF) set(TESTING OFF) endif() +set(MULTITHREADING OFF) if(CMAKE_SYSTEM_PROCESSOR MATCHES "wasm32") message(STATUS "Compiling for WebAssembly.") diff --git a/cpp/src/barretenberg/common/msgpack-cbind.hpp b/cpp/src/barretenberg/common/msgpack-cbind.hpp new file mode 100644 index 0000000000..1989bd1dcd --- /dev/null +++ b/cpp/src/barretenberg/common/msgpack-cbind.hpp @@ -0,0 +1,34 @@ +#pragma once +// Note: heavy header due to serialization logic, don't include if auto parameters will do +#include "msgpack-impl.hpp" + +namespace msgpack { +inline std::pair encode_buffer(auto obj) +{ + // Create a buffer to store the encoded data + msgpack::sbuffer buffer; + msgpack::pack(buffer, obj); + + uint8_t* output = (uint8_t*)aligned_alloc(64, buffer.size()); + memcpy(output, buffer.data(), buffer.size()); + // Convert the buffer data to a string and return it + return { output, buffer.size() }; +} + +template inline void decode(T* value, const uint8_t* encoded_data, size_t encoded_data_size) +{ + // Create a MsgPack unpacker + msgpack::unpack((const char*)encoded_data, encoded_data_size).get().convert(*value); +} + +void print(const auto& obj) +{ + std::stringstream output; + msgpack::pack(output, obj); + std::string output_str = output.str(); + msgpack::object_handle oh = msgpack::unpack(output_str.data(), output_str.size()); + std::stringstream pretty_output; + pretty_output << oh.get() << std::endl; + logstr(pretty_output.str().c_str()); +} +} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/srs/msgpack-impl.hpp b/cpp/src/barretenberg/common/msgpack-impl.hpp similarity index 77% rename from cpp/src/barretenberg/srs/msgpack-impl.hpp rename to cpp/src/barretenberg/common/msgpack-impl.hpp index a3f6efb66e..bbd6dee4bb 100644 --- a/cpp/src/barretenberg/srs/msgpack-impl.hpp +++ b/cpp/src/barretenberg/common/msgpack-impl.hpp @@ -1,6 +1,5 @@ #pragma once -// Meant to be included by cbind compilation units. -// Avoid including in headers as it msgpack.hpp includes a good deal of headers. +// Note: heavy header due to serialization logic, don't include if auto parameters will do #include #include #include @@ -68,16 +67,13 @@ template struct pack { } }; -// template struct object_with_zone { -// void operator()(msgpack::object::with_zone& o, T const& v) const -// { -// const_cast(v).msgpack(DefineMapArchive{}).msgpack_object(&o, o.zone); -// } -// }; -// template struct object_with_zone { -// void operator()(msgpack::object::with_zone& o, T const& v) const -// { -// const_cast(v).msgpack_flat(DefineArchive{}).msgpack_object(o); -// } -// }; +template struct pack> { + template + msgpack::packer& operator()(msgpack::packer& o, std::variant const& variant) const + { + std::visit([&o](auto&& arg) { msgpack::pack(o, arg); }, variant); + return o; + } +}; + } // namespace msgpack::adaptor diff --git a/cpp/src/barretenberg/common/msgpack-schema.hpp b/cpp/src/barretenberg/common/msgpack-schema.hpp new file mode 100644 index 0000000000..4ad5e7f8e2 --- /dev/null +++ b/cpp/src/barretenberg/common/msgpack-schema.hpp @@ -0,0 +1,105 @@ +#pragma once + +#define MSGPACK_NO_BOOST +#include "msgpack-cbind.hpp" +#include + +namespace msgpack { + +template +concept TupleLike = requires(T a) { + std::tuple_size::value; + std::get<0>(a); + }; + +template struct MsgPackSchema; + +struct DefineMapSchema { + auto operator()() { return std::make_tuple(); } + + auto operator()(auto key, auto value) + { + (void)value; // unused except for type + return std::make_tuple(key, MsgPackSchema()); + } + + auto operator()(auto key, auto value, auto&... args) + { + (void)value; // unused except for type + return std::tuple_cat(std::make_tuple(key, MsgPackSchema()), (*this)(args...)); + } +}; + +template std::string schema_name() +{ + std::string result = abi::__cxa_demangle(typeid(T).name(), NULL, NULL, NULL); + if (result.find("basic_string") != std::string::npos) { + return "string"; + } + if (result == "i") { + return "int"; + } + if (result.find('<') != size_t(-1)) { + result = result.substr(0, result.find('<')); + } + if (result.rfind(':') != size_t(-1)) { + result = result.substr(result.rfind(':') + 1, result.size()); + } + return result; +} + +template struct MsgPackSchema { + // A viable reference is needed for define_map + const char* type_name_string = "__typename"; + std::string type_name = schema_name(); + + void msgpack_pack(auto& packer) const + requires msgpack::adaptor::HasMsgPack + { + T object; + object.msgpack([&](auto&... args) { + msgpack::adaptor::DefineMapArchive ar; + auto archive_wrapper = [&](auto&... args) { return ar(type_name_string, type_name, args...); }; + auto pack_map_schema = [&](auto&... args) { + auto schema = DefineMapSchema{}(args...); + std::apply(archive_wrapper, schema).msgpack_pack(packer); + }; + std::apply(pack_map_schema, ar(args...).a); + }); + } + + void msgpack_pack(auto& packer) const + requires msgpack::adaptor::HasMsgPackFlat + { + T object; + object.msgpack_flat([&](auto&... args) { + MsgPackSchema schema{ ar(args...).a }; + schema.msg_pack(packer); + }); + } + + void msgpack_pack(auto& packer) const + requires TupleLike + { + T object; + msgpack::adaptor::DefineArchive ar; + auto archive_wrapper = [&](auto&... args) { return ar(type_name, args...); }; + auto pack_array_schema = [&](auto&... args) { + auto schema = std::make_tuple(MsgPackSchema>{}...); + std::apply(archive_wrapper, schema).msgpack_pack(packer); + }; + std::apply(pack_array_schema, object); + } + + void msgpack_pack(auto& packer) const + requires(!TupleLike && !msgpack::adaptor::HasMsgPack && !msgpack::adaptor::HasMsgPackFlat) + { + packer.pack_str((uint32_t)type_name.size()); + packer.pack_str_body(type_name.c_str(), (uint32_t)type_name.size()); + } +}; +void print_schema(auto obj) +{ + msgpack::print(MsgPackSchema{}); +} +} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/ecc/fields/field.hpp b/cpp/src/barretenberg/ecc/fields/field.hpp index 66c59ea0d2..a7d61d5d5f 100644 --- a/cpp/src/barretenberg/ecc/fields/field.hpp +++ b/cpp/src/barretenberg/ecc/fields/field.hpp @@ -402,6 +402,10 @@ template struct alignas(32) field { // BBERG_INLINE sstatic constexpr void butterfly(field& left, field& right) noexcept; + // For serialization + void msgpack_pack(auto& packer) const; + void msgpack_unpack(auto o); + private: static constexpr uint256_t twice_modulus = modulus + modulus; static constexpr uint256_t not_modulus = -modulus; @@ -539,7 +543,6 @@ template void read(B& it, field& value) read(it, result.data[0]); value = result.to_montgomery_form(); } - template void write(B& buf, field const& value) { using serialize::write; diff --git a/cpp/src/barretenberg/ecc/fields/field_impl.hpp b/cpp/src/barretenberg/ecc/fields/field_impl.hpp index 057a7e4463..54ba752b54 100644 --- a/cpp/src/barretenberg/ecc/fields/field_impl.hpp +++ b/cpp/src/barretenberg/ecc/fields/field_impl.hpp @@ -13,6 +13,12 @@ #include "field_impl_generic.hpp" +// For serialization +namespace msgpack::v2 { // Hack, have to forward v2 namespace +struct object; +} +void msgpack_read_bin64(const msgpack::v2::object& obj, uint64_t* data, size_t size); + namespace barretenberg { // template constexpr void field::butterfly(field& left, field& right) noexcept @@ -607,4 +613,22 @@ template constexpr field field::multiplicative_generator() noexc return target; } +// For serialization +template void field::msgpack_pack(auto& packer) const +{ + auto adjusted = from_montgomery_form(); + uint64_t bin_data[4] = { + htonll(adjusted.data[0]), htonll(adjusted.data[1]), htonll(adjusted.data[2]), htonll(adjusted.data[3]) + }; + packer.pack_bin(sizeof(bin_data)); + packer.pack_bin_body((const char*)bin_data, sizeof(bin_data)); +} + +// For serialization +template void field::msgpack_unpack(auto o) +{ + msgpack_read_bin64(o, data, sizeof(data) / sizeof(uint64_t)); + *this = to_montgomery_form(); +} + } // namespace barretenberg diff --git a/cpp/src/barretenberg/ecc/fields/msgpack.cpp b/cpp/src/barretenberg/ecc/fields/msgpack.cpp new file mode 100644 index 0000000000..b66b9771df --- /dev/null +++ b/cpp/src/barretenberg/ecc/fields/msgpack.cpp @@ -0,0 +1,20 @@ +#define MSGPACK_NO_BOOST +#include +#include +#include +#include + +void msgpack_read_bin64(const msgpack::object& obj, unsigned long long* data, size_t size) +{ + if (obj.type != msgpack::type::BIN) { + throw_or_abort("Wrong data type when unpacking bin64"); + } + if (obj.via.bin.size != size * sizeof(unsigned long long)) { + throw_or_abort("Wrong size data while unpacking bin64"); + } + + unsigned long long* bin_data = (uint64_t*)obj.via.bin.ptr; + for (size_t i = 0; i < size; i++) { + data[i] = ntohll(bin_data[i]); + } +} diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index 937c758648..ac419cee16 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -7,7 +7,6 @@ #include "msgpack.hpp" #include "msgpack-impl.hpp" #include "barretenberg/crypto/aes128/aes128.hpp" -#include using namespace barretenberg; @@ -51,76 +50,6 @@ struct MyExampleMap { void msgpack(auto ar) { ar(NVP(a), NVP(b), NVP(flat)); } }; -template struct MsgPackSchema; - -struct DefineMapSchema { - auto operator()() { return std::make_tuple(); } - auto operator()(auto key, auto value) - { - (void)value; // unused except for type - return std::make_tuple(key, MsgPackSchema()); - } - - auto operator()(auto key, auto value, auto&... args) - { - (void)value; // unused except for type - return std::tuple_cat(std::make_tuple(key, MsgPackSchema()), (*this)(args...)); - } -}; - -template const char* schema_name() -{ - const char* result = abi::__cxa_demangle(typeid(T).name(), NULL, NULL, NULL); - std::string result_str = result; - if (result_str.find("basic_string") != std::string::npos) { - return "string"; - } - if (result_str == "i") { - return "int"; - } - return result; -} - -template struct MsgPackSchema { - // A viable reference is needed for define_map - const char* type_name_string = "__typename"; - const char* type_name = schema_name(); - void msgpack_pack(auto& packer) const - requires msgpack::adaptor::HasMsgPack - { - T object; - object.msgpack([&](auto&... args) { - msgpack::adaptor::DefineMapArchive ar; - auto archive_wrapper = [&](auto&... args) { return ar(type_name_string, type_name, args...); }; - auto pack_map_schema = [&](auto&... args) { - auto schema = DefineMapSchema{}(args...); - std::apply(archive_wrapper, schema).msgpack_pack(packer); - }; - std::apply(pack_map_schema, ar(args...).a); - }); - } - void msgpack_pack(auto& packer) const - requires msgpack::adaptor::HasMsgPackFlat - { - T object; - object.msgpack_flat([&](auto&... args) { - msgpack::adaptor::DefineArchive ar; - auto archive_wrapper = [&](auto&... args) { return ar(type_name, args...); }; - auto pack_array_schema = [&](auto&... args) { - auto schema = std::make_tuple(MsgPackSchema>{}...); - std::apply(archive_wrapper, schema).msgpack_pack(packer); - }; - std::apply(pack_array_schema, ar(args...).a); - }); - } - void msgpack_pack(auto& packer) const - requires(!msgpack::adaptor::HasMsgPack && !msgpack::adaptor::HasMsgPackFlat) - { - packer.pack_str((uint32_t)strlen(type_name)); - packer.pack_str_body(type_name, (uint32_t)strlen(type_name)); - } -}; - namespace cbinds { struct aes__decrypt_buffer_cbc { std::vector in; diff --git a/cpp/src/barretenberg/stdlib/primitives/address/address.hpp b/cpp/src/barretenberg/stdlib/primitives/address/address.hpp index 0b27a443c6..d187b96de8 100644 --- a/cpp/src/barretenberg/stdlib/primitives/address/address.hpp +++ b/cpp/src/barretenberg/stdlib/primitives/address/address.hpp @@ -47,6 +47,9 @@ class address { friend std::ostream& operator<<(std::ostream& os, address const& v) { return os << v.address_; } fr to_field() const { return address_; } + + void msgpack_pack(auto& packer) const { address_.msgpack_pack(packer); } + void msgpack_unpack(auto const& o) { address_.msgpack_unpack(o); } }; template void read(B& it, address& addr) From b76b7b2b1697b647f0d733d1a5580f03134c7092 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 08:49:58 -0400 Subject: [PATCH 22/92] fix: fork msgpack for greater checks --- .gitmodules | 2 +- cpp/src/msgpack-c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 31154308d0..dcb2e83d51 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = git@github.com:AztecProtocol/foundation.git [submodule "cpp/src/msgpack-c"] path = cpp/src/msgpack-c - url = https://github.com/msgpack/msgpack-c + url = https://github.com/AztecProtocol/msgpack-c diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index 1edfba0cd2..6fe406e9be 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit 1edfba0cd24378bba74578f18ca96e3809ae1664 +Subproject commit 6fe406e9be95c7eee22db4f0e42651f660b4a65d From 4f685b351544f937eb239c2d8fc4fbff089635ed Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 09:16:59 -0400 Subject: [PATCH 23/92] Refactor --- .circleci/config.yml | 1 + cpp/src/barretenberg/common/msgpack-cbind.hpp | 34 ------ .../barretenberg/common/msgpack-schema.hpp | 105 ------------------ .../msgpack.cpp => common/msgpack_bin.cpp} | 10 +- cpp/src/barretenberg/common/msgpack_bin.hpp | 13 +++ .../{msgpack-impl.hpp => msgpack_impl.hpp} | 0 .../barretenberg/ecc/fields/field_impl.hpp | 8 +- cpp/src/barretenberg/srs/io.test.cpp | 2 +- 8 files changed, 23 insertions(+), 150 deletions(-) delete mode 100644 cpp/src/barretenberg/common/msgpack-cbind.hpp delete mode 100644 cpp/src/barretenberg/common/msgpack-schema.hpp rename cpp/src/barretenberg/{ecc/fields/msgpack.cpp => common/msgpack_bin.cpp} (65%) create mode 100644 cpp/src/barretenberg/common/msgpack_bin.hpp rename cpp/src/barretenberg/common/{msgpack-impl.hpp => msgpack_impl.hpp} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4618ddcca..d2be880902 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,6 +49,7 @@ checkout: &checkout git remote add origin $CIRCLE_REPOSITORY_URL git fetch --depth 50 --filter=blob:none origin $CIRCLE_SHA1 git checkout FETCH_HEAD + git submodule update --init --recursive # This build step checks out the code from the benchmark-archive repository. The key is saved in CircleCi environment in base64 format. # Initially it just fetches the latest version. diff --git a/cpp/src/barretenberg/common/msgpack-cbind.hpp b/cpp/src/barretenberg/common/msgpack-cbind.hpp deleted file mode 100644 index 1989bd1dcd..0000000000 --- a/cpp/src/barretenberg/common/msgpack-cbind.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once -// Note: heavy header due to serialization logic, don't include if auto parameters will do -#include "msgpack-impl.hpp" - -namespace msgpack { -inline std::pair encode_buffer(auto obj) -{ - // Create a buffer to store the encoded data - msgpack::sbuffer buffer; - msgpack::pack(buffer, obj); - - uint8_t* output = (uint8_t*)aligned_alloc(64, buffer.size()); - memcpy(output, buffer.data(), buffer.size()); - // Convert the buffer data to a string and return it - return { output, buffer.size() }; -} - -template inline void decode(T* value, const uint8_t* encoded_data, size_t encoded_data_size) -{ - // Create a MsgPack unpacker - msgpack::unpack((const char*)encoded_data, encoded_data_size).get().convert(*value); -} - -void print(const auto& obj) -{ - std::stringstream output; - msgpack::pack(output, obj); - std::string output_str = output.str(); - msgpack::object_handle oh = msgpack::unpack(output_str.data(), output_str.size()); - std::stringstream pretty_output; - pretty_output << oh.get() << std::endl; - logstr(pretty_output.str().c_str()); -} -} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/common/msgpack-schema.hpp b/cpp/src/barretenberg/common/msgpack-schema.hpp deleted file mode 100644 index 4ad5e7f8e2..0000000000 --- a/cpp/src/barretenberg/common/msgpack-schema.hpp +++ /dev/null @@ -1,105 +0,0 @@ -#pragma once - -#define MSGPACK_NO_BOOST -#include "msgpack-cbind.hpp" -#include - -namespace msgpack { - -template -concept TupleLike = requires(T a) { - std::tuple_size::value; - std::get<0>(a); - }; - -template struct MsgPackSchema; - -struct DefineMapSchema { - auto operator()() { return std::make_tuple(); } - - auto operator()(auto key, auto value) - { - (void)value; // unused except for type - return std::make_tuple(key, MsgPackSchema()); - } - - auto operator()(auto key, auto value, auto&... args) - { - (void)value; // unused except for type - return std::tuple_cat(std::make_tuple(key, MsgPackSchema()), (*this)(args...)); - } -}; - -template std::string schema_name() -{ - std::string result = abi::__cxa_demangle(typeid(T).name(), NULL, NULL, NULL); - if (result.find("basic_string") != std::string::npos) { - return "string"; - } - if (result == "i") { - return "int"; - } - if (result.find('<') != size_t(-1)) { - result = result.substr(0, result.find('<')); - } - if (result.rfind(':') != size_t(-1)) { - result = result.substr(result.rfind(':') + 1, result.size()); - } - return result; -} - -template struct MsgPackSchema { - // A viable reference is needed for define_map - const char* type_name_string = "__typename"; - std::string type_name = schema_name(); - - void msgpack_pack(auto& packer) const - requires msgpack::adaptor::HasMsgPack - { - T object; - object.msgpack([&](auto&... args) { - msgpack::adaptor::DefineMapArchive ar; - auto archive_wrapper = [&](auto&... args) { return ar(type_name_string, type_name, args...); }; - auto pack_map_schema = [&](auto&... args) { - auto schema = DefineMapSchema{}(args...); - std::apply(archive_wrapper, schema).msgpack_pack(packer); - }; - std::apply(pack_map_schema, ar(args...).a); - }); - } - - void msgpack_pack(auto& packer) const - requires msgpack::adaptor::HasMsgPackFlat - { - T object; - object.msgpack_flat([&](auto&... args) { - MsgPackSchema schema{ ar(args...).a }; - schema.msg_pack(packer); - }); - } - - void msgpack_pack(auto& packer) const - requires TupleLike - { - T object; - msgpack::adaptor::DefineArchive ar; - auto archive_wrapper = [&](auto&... args) { return ar(type_name, args...); }; - auto pack_array_schema = [&](auto&... args) { - auto schema = std::make_tuple(MsgPackSchema>{}...); - std::apply(archive_wrapper, schema).msgpack_pack(packer); - }; - std::apply(pack_array_schema, object); - } - - void msgpack_pack(auto& packer) const - requires(!TupleLike && !msgpack::adaptor::HasMsgPack && !msgpack::adaptor::HasMsgPackFlat) - { - packer.pack_str((uint32_t)type_name.size()); - packer.pack_str_body(type_name.c_str(), (uint32_t)type_name.size()); - } -}; -void print_schema(auto obj) -{ - msgpack::print(MsgPackSchema{}); -} -} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/ecc/fields/msgpack.cpp b/cpp/src/barretenberg/common/msgpack_bin.cpp similarity index 65% rename from cpp/src/barretenberg/ecc/fields/msgpack.cpp rename to cpp/src/barretenberg/common/msgpack_bin.cpp index b66b9771df..0c116a232e 100644 --- a/cpp/src/barretenberg/ecc/fields/msgpack.cpp +++ b/cpp/src/barretenberg/common/msgpack_bin.cpp @@ -4,17 +4,21 @@ #include #include -void msgpack_read_bin64(const msgpack::object& obj, unsigned long long* data, size_t size) +namespace msgpack { + +void read_bin64(const msgpack::object& obj, uint64_t* data, size_t size) { if (obj.type != msgpack::type::BIN) { throw_or_abort("Wrong data type when unpacking bin64"); } - if (obj.via.bin.size != size * sizeof(unsigned long long)) { + if (obj.via.bin.size != size * sizeof(uint64_t) { throw_or_abort("Wrong size data while unpacking bin64"); } - unsigned long long* bin_data = (uint64_t*)obj.via.bin.ptr; + uint64_t* bin_data = (uint64_t*)obj.via.bin.ptr; for (size_t i = 0; i < size; i++) { data[i] = ntohll(bin_data[i]); } } + +} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/common/msgpack_bin.hpp b/cpp/src/barretenberg/common/msgpack_bin.hpp new file mode 100644 index 0000000000..e9c8291e8f --- /dev/null +++ b/cpp/src/barretenberg/common/msgpack_bin.hpp @@ -0,0 +1,13 @@ +#include +#include + +namespace msgpack::v2 { // Hack, have to forward v2 namespace +struct object; +} + +// For field serialization +namespace msgpack { + +void read_bin64(const msgpack::v2::object& obj, uint64_t* data, size_t size); + +} diff --git a/cpp/src/barretenberg/common/msgpack-impl.hpp b/cpp/src/barretenberg/common/msgpack_impl.hpp similarity index 100% rename from cpp/src/barretenberg/common/msgpack-impl.hpp rename to cpp/src/barretenberg/common/msgpack_impl.hpp diff --git a/cpp/src/barretenberg/ecc/fields/field_impl.hpp b/cpp/src/barretenberg/ecc/fields/field_impl.hpp index 54ba752b54..3a53b26738 100644 --- a/cpp/src/barretenberg/ecc/fields/field_impl.hpp +++ b/cpp/src/barretenberg/ecc/fields/field_impl.hpp @@ -12,13 +12,7 @@ #endif #include "field_impl_generic.hpp" - -// For serialization -namespace msgpack::v2 { // Hack, have to forward v2 namespace -struct object; -} -void msgpack_read_bin64(const msgpack::v2::object& obj, uint64_t* data, size_t size); - +#include "barretenberg/common/msgpack_bin.hpp" namespace barretenberg { // template constexpr void field::butterfly(field& left, field& right) noexcept diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index ac419cee16..a2098bdf82 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -5,7 +5,7 @@ #include #include #include "msgpack.hpp" -#include "msgpack-impl.hpp" +#include "msgpack_impl.hpp" #include "barretenberg/crypto/aes128/aes128.hpp" using namespace barretenberg; From 566264099d192ca1f73a40832bfd150c67a2c417 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 09:20:00 -0400 Subject: [PATCH 24/92] cleanup --- barretenberg.code-workspace | 3 +-- cpp/src/barretenberg/honk/transcript/transcript.hpp | 2 +- .../proof_system/polynomial_store/polynomial_store.test.cpp | 2 +- cpp/src/msgpack-c | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/barretenberg.code-workspace b/barretenberg.code-workspace index 823190927d..c0737c9176 100644 --- a/barretenberg.code-workspace +++ b/barretenberg.code-workspace @@ -149,8 +149,7 @@ "cmake.configureArgs": [ "--preset clang15", "-G Ninja", - "-g", - "-DMULTITHREADING=OFF" + "-g" ], "cmake.useCMakePresets": "auto", }, diff --git a/cpp/src/barretenberg/honk/transcript/transcript.hpp b/cpp/src/barretenberg/honk/transcript/transcript.hpp index 86cd4a1ee2..ff859806b2 100644 --- a/cpp/src/barretenberg/honk/transcript/transcript.hpp +++ b/cpp/src/barretenberg/honk/transcript/transcript.hpp @@ -191,7 +191,7 @@ template class ProverTranscript : public BaseTranscript { * @details Serializes the provided object into `proof_data`, and updates the current round state. * * @param label Description/name of the object being added. - * @param element HasMsgPackFlat object that will be added to the transcript + * @param element Serializable object that will be added to the transcript * * @todo Use a concept to only allow certain types to be passed. Requirements are that the object should be * serializable. diff --git a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp index 38b3138655..a068cce790 100644 --- a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp +++ b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp @@ -102,7 +102,7 @@ TEST(PolynomialStore, RangeBasedFor) polynomial_store.put("id_2", std::move(poly2)); // Check that PolynomialStore meets criteria for std::ranges::range - // EXPECT_TRUE(std::ranges::range>); + EXPECT_TRUE(std::ranges::range>); // For example ... Polynomial polynomial_sum(size); diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index 6fe406e9be..a7de6a02c8 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit 6fe406e9be95c7eee22db4f0e42651f660b4a65d +Subproject commit a7de6a02c88bd7950757ea8427235f81b5f9f64d From 53e0e1bb0bdb9bfc39d91b30fcd9dc6d1cbea71d Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 09:20:54 -0400 Subject: [PATCH 25/92] Update turbo_circuit_constructor.cpp --- .../circuit_constructors/turbo_circuit_constructor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp b/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp index 1e2fd973d1..517587f750 100644 --- a/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp +++ b/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp @@ -1448,7 +1448,7 @@ inline fr TurboCircuitConstructor::fixed_base_gate_evaluation(const size_t gate_ * */ bool TurboCircuitConstructor::check_circuit() { -// #define LAZY_CIRCUIT_CHECKS +//#define LAZY_CIRCUIT_CHECKS #ifdef LAZY_CIRCUIT_CHECKS for (size_t i = 0; i < num_gates; i++) { if (!q_arith[i].is_zero() && !lazy_arithmetic_gate_check(i)) { From 2322e8209b101f16a7aa53f2b0f49b971917175f Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 10:14:17 -0400 Subject: [PATCH 26/92] chore: continued cleanup --- .circleci/config.yml | 1 - .../{common/msgpack_bin.cpp => ecc/fields/_msgpack_bin.cpp} | 2 +- .../{common/msgpack_bin.hpp => ecc/fields/_msgpack_bin.hpp} | 4 +++- cpp/src/barretenberg/ecc/fields/field_impl.hpp | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) rename cpp/src/barretenberg/{common/msgpack_bin.cpp => ecc/fields/_msgpack_bin.cpp} (91%) rename cpp/src/barretenberg/{common/msgpack_bin.hpp => ecc/fields/_msgpack_bin.hpp} (65%) diff --git a/.circleci/config.yml b/.circleci/config.yml index d2be880902..c4618ddcca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,6 @@ checkout: &checkout git remote add origin $CIRCLE_REPOSITORY_URL git fetch --depth 50 --filter=blob:none origin $CIRCLE_SHA1 git checkout FETCH_HEAD - git submodule update --init --recursive # This build step checks out the code from the benchmark-archive repository. The key is saved in CircleCi environment in base64 format. # Initially it just fetches the latest version. diff --git a/cpp/src/barretenberg/common/msgpack_bin.cpp b/cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp similarity index 91% rename from cpp/src/barretenberg/common/msgpack_bin.cpp rename to cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp index 0c116a232e..d300087a00 100644 --- a/cpp/src/barretenberg/common/msgpack_bin.cpp +++ b/cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp @@ -11,7 +11,7 @@ void read_bin64(const msgpack::object& obj, uint64_t* data, size_t size) if (obj.type != msgpack::type::BIN) { throw_or_abort("Wrong data type when unpacking bin64"); } - if (obj.via.bin.size != size * sizeof(uint64_t) { + if (obj.via.bin.size != size * sizeof(uint64_t)) { throw_or_abort("Wrong size data while unpacking bin64"); } diff --git a/cpp/src/barretenberg/common/msgpack_bin.hpp b/cpp/src/barretenberg/ecc/fields/_msgpack_bin.hpp similarity index 65% rename from cpp/src/barretenberg/common/msgpack_bin.hpp rename to cpp/src/barretenberg/ecc/fields/_msgpack_bin.hpp index e9c8291e8f..5557516863 100644 --- a/cpp/src/barretenberg/common/msgpack_bin.hpp +++ b/cpp/src/barretenberg/ecc/fields/_msgpack_bin.hpp @@ -1,11 +1,13 @@ #include #include +// Implementation detail for field serialization +// We are very careful to include msgpack.hpp only in cpp units + namespace msgpack::v2 { // Hack, have to forward v2 namespace struct object; } -// For field serialization namespace msgpack { void read_bin64(const msgpack::v2::object& obj, uint64_t* data, size_t size); diff --git a/cpp/src/barretenberg/ecc/fields/field_impl.hpp b/cpp/src/barretenberg/ecc/fields/field_impl.hpp index 3a53b26738..ec447d02a9 100644 --- a/cpp/src/barretenberg/ecc/fields/field_impl.hpp +++ b/cpp/src/barretenberg/ecc/fields/field_impl.hpp @@ -12,7 +12,7 @@ #endif #include "field_impl_generic.hpp" -#include "barretenberg/common/msgpack_bin.hpp" +#include "_msgpack_bin.hpp" namespace barretenberg { // template constexpr void field::butterfly(field& left, field& right) noexcept @@ -621,7 +621,7 @@ template void field::msgpack_pack(auto& packer) const // For serialization template void field::msgpack_unpack(auto o) { - msgpack_read_bin64(o, data, sizeof(data) / sizeof(uint64_t)); + msgpack::read_bin64(o, data, sizeof(data) / sizeof(uint64_t)); *this = to_montgomery_form(); } From 70aff201f19774a95b59517b105b6941d7941319 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 10:22:28 -0400 Subject: [PATCH 27/92] chore: continued cleanup --- cpp/src/barretenberg/env/crs.cpp | 10 +-- .../plonk/proof_system/types/proof.hpp | 2 +- cpp/src/barretenberg/srs/io.test.cpp | 75 ------------------- 3 files changed, 6 insertions(+), 81 deletions(-) diff --git a/cpp/src/barretenberg/env/crs.cpp b/cpp/src/barretenberg/env/crs.cpp index 923435057d..80561d1c09 100644 --- a/cpp/src/barretenberg/env/crs.cpp +++ b/cpp/src/barretenberg/env/crs.cpp @@ -7,13 +7,13 @@ const int NUM_POINTS_IN_TRANSCRIPT = 5040001; + extern "C" { /** * @brief In WASM, loads the verifier reference string. * Used in native code to quickly create an in-memory reference string. */ -uint8_t* env_load_verifier_crs() -{ +uint8_t* env_load_verifier_crs() { std::ifstream transcript; transcript.open("../srs_db/ignition/monomial/transcript00.dat", std::ifstream::binary); // We need two g2 points, each 64 bytes. @@ -33,14 +33,13 @@ uint8_t* env_load_verifier_crs() * In native code, not intended to be used. * @param num_points The number of points to load. */ -uint8_t* env_load_prover_crs(size_t num_points) -{ +uint8_t* env_load_prover_crs(size_t num_points) { // Note: This implementation is only meant to be instructive. // This should only be used in c-binds to implement the C++ abstractions. std::ifstream transcript; transcript.open("../srs_db/ignition/monomial/transcript00.dat", std::ifstream::binary); // Each g1 point is 64 bytes. - size_t g1_points_size = (num_points)*64; + size_t g1_points_size = (num_points) * 64; std::vector g1_points(g1_points_size); transcript.seekg(28); transcript.read((char*)g1_points.data(), (std::streamsize)g1_points_size); @@ -49,4 +48,5 @@ uint8_t* env_load_prover_crs(size_t num_points) memcpy(g1_points_copy, g1_points.data(), g1_points_size); return g1_points_copy; } + } \ No newline at end of file diff --git a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp index d0a65aa2b6..aedf516e04 100644 --- a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp @@ -28,7 +28,7 @@ template inline void write(B& buf, proof const& data) inline std::ostream& operator<<(std::ostream& os, proof const& data) { // REFACTOR: This is copied from barretenberg/common/streams.hpp, - // which means we could just cout proof_data directly, but that breaks the build in the CI with + // which means we could just cout proof_data directly, but that breaks the build in the CI with // a redefined operator<< error in barretenberg/stdlib/hash/keccak/keccak.test.cpp, // which is something we really don't want to deal with right now. std::ios_base::fmtflags f(os.flags()); diff --git a/cpp/src/barretenberg/srs/io.test.cpp b/cpp/src/barretenberg/srs/io.test.cpp index a2098bdf82..7ff2806227 100644 --- a/cpp/src/barretenberg/srs/io.test.cpp +++ b/cpp/src/barretenberg/srs/io.test.cpp @@ -3,10 +3,6 @@ #include "io.hpp" #include "barretenberg/common/mem.hpp" #include -#include -#include "msgpack.hpp" -#include "msgpack_impl.hpp" -#include "barretenberg/crypto/aes128/aes128.hpp" using namespace barretenberg; @@ -34,74 +30,3 @@ TEST(io, read_transcript_loads_well_formed_srs) } aligned_free(monomials); } - -struct MyExampleFlat { - int a; - std::string b; - - void msgpack_flat(auto ar) { ar(a, b); } -}; - -struct MyExampleMap { - int a; - std::string b; - MyExampleFlat flat; - - void msgpack(auto ar) { ar(NVP(a), NVP(b), NVP(flat)); } -}; - -namespace cbinds { -struct aes__decrypt_buffer_cbc { - std::vector in; - std::vector iv; - std::vector key; - size_t length; - void msgpack_flat(auto ar) { ar(in, iv, key, length); } - auto operator()() - { - crypto::aes128::decrypt_buffer_cbc(in.data(), iv.data(), key.data(), length); - return in; - } -}; -} // namespace cbinds - -// auto aes__decrypt_buffer_cbc(uint8_t* in, uint8_t* iv, const uint8_t* key, const size_t length, uint8_t* r) {} -// auto cbind_example() {} - -void pretty_print(const auto& obj) -{ - std::stringstream output; - msgpack::pack(output, obj); - std::string output_str = output.str(); - msgpack::object_handle oh = msgpack::unpack(output_str.data(), output_str.size()); - std::cout << oh.get() << std::endl; -} - -TEST(io, myexample) -{ - - { // pack, unpack - MyExampleMap my{ 1, "2", { 3, "4" } }; - MsgPackSchema my_schema; - pretty_print(my); - pretty_print(my_schema); - std::stringstream ss; - msgpack::pack(ss, my); - - std::string const& str = ss.str(); - // Write the packed data to a file - std::ofstream ofs("output.msgpack", std::ios::binary); - if (ofs) { - ofs.write(str.data(), (std::streamsize)str.size()); - ofs.close(); - std::cout << "Binary string written to output.msgpack" << std::endl; - } else { - std::cerr << "Error: Unable to open output.msgpack" << std::endl; - } - msgpack::object_handle oh = msgpack::unpack(str.data(), str.size()); - msgpack::object obj = oh.get(); - std::cout << obj << std::endl; - MyExampleMap map = obj.convert(); - std::cout << map.b << std::endl; - } -} \ No newline at end of file From 3a22e062c458be8f21d22b868e12d72201986ec8 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 10:25:07 -0400 Subject: [PATCH 28/92] chore: continued cleanup --- cpp/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0177dcf954..582d728537 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -74,7 +74,6 @@ if(FUZZING) set(MULTITHREADING OFF) set(TESTING OFF) endif() -set(MULTITHREADING OFF) if(CMAKE_SYSTEM_PROCESSOR MATCHES "wasm32") message(STATUS "Compiling for WebAssembly.") From 8b5e22465ea51def078fe5b5aa77a9e515efcc83 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 10:25:41 -0400 Subject: [PATCH 29/92] Refactor --- cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp index f503481ce7..1b58491c3c 100644 --- a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp +++ b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp @@ -30,6 +30,7 @@ WASM_EXPORT void pedersen_plookup_compress_fields(uint8_t const* left, uint8_t c barretenberg::fr::serialize_to_buffer(r, result); } + WASM_EXPORT void pedersen__compress(uint8_t const* inputs_buffer, uint8_t* output) { std::vector to_compress; From 0ebf8f5c88a78cb959b598f664884f42fffab9bc Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 10:27:14 -0400 Subject: [PATCH 30/92] Refactor --- cpp/src/barretenberg/env/crs.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/src/barretenberg/env/crs.hpp b/cpp/src/barretenberg/env/crs.hpp index 871fb817d5..6a92c92120 100644 --- a/cpp/src/barretenberg/env/crs.hpp +++ b/cpp/src/barretenberg/env/crs.hpp @@ -2,9 +2,9 @@ #include // To be provided by the environment. -// Outputs from a trusted setup "Common reference string" model: -// https://en.wikipedia.org/wiki/Common_reference_string_model For a WASM build, this is provided by the JavaScript -// environment. For a native build, this is provided in this module. +// Outputs from a trusted setup "Common reference string" model: https://en.wikipedia.org/wiki/Common_reference_string_model +// For a WASM build, this is provided by the JavaScript environment. +// For a native build, this is provided in this module. /** * @brief In WASM, loads the verifier reference string. From 47c4a1f6c55844d5cd0dad8769fff169688f1b5e Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 10:46:55 -0400 Subject: [PATCH 31/92] fix: ci --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4618ddcca..d2be880902 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,6 +49,7 @@ checkout: &checkout git remote add origin $CIRCLE_REPOSITORY_URL git fetch --depth 50 --filter=blob:none origin $CIRCLE_SHA1 git checkout FETCH_HEAD + git submodule update --init --recursive # This build step checks out the code from the benchmark-archive repository. The key is saved in CircleCi environment in base64 format. # Initially it just fetches the latest version. From 3fb25dc53aa82f455830d881652c523a7b9c1137 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 14:44:23 -0400 Subject: [PATCH 32/92] feat(wasm): hacks to make work in a fno-exceptions wasm environment --- cpp/src/barretenberg/common/msgpack_impl.hpp | 1 + cpp/src/barretenberg/common/throw_or_abort.hpp | 2 +- cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp | 6 +++--- cpp/src/msgpack-c | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cpp/src/barretenberg/common/msgpack_impl.hpp b/cpp/src/barretenberg/common/msgpack_impl.hpp index bbd6dee4bb..3f1db7e49d 100644 --- a/cpp/src/barretenberg/common/msgpack_impl.hpp +++ b/cpp/src/barretenberg/common/msgpack_impl.hpp @@ -5,6 +5,7 @@ #include #include #include +#include "throw_or_abort.hpp" #define MSGPACK_NO_BOOST #include diff --git a/cpp/src/barretenberg/common/throw_or_abort.hpp b/cpp/src/barretenberg/common/throw_or_abort.hpp index 02f57437ce..dcaad88433 100644 --- a/cpp/src/barretenberg/common/throw_or_abort.hpp +++ b/cpp/src/barretenberg/common/throw_or_abort.hpp @@ -10,4 +10,4 @@ inline void throw_or_abort [[noreturn]] (std::string const& err) info(err); std::abort(); #endif -} +} \ No newline at end of file diff --git a/cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp b/cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp index d300087a00..a626ea01ef 100644 --- a/cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp +++ b/cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp @@ -1,9 +1,9 @@ + +#include #define MSGPACK_NO_BOOST #include #include #include -#include - namespace msgpack { void read_bin64(const msgpack::object& obj, uint64_t* data, size_t size) @@ -21,4 +21,4 @@ void read_bin64(const msgpack::object& obj, uint64_t* data, size_t size) } } -} // namespace msgpack \ No newline at end of file +} // namespace msgpack diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index a7de6a02c8..55fc2d49c3 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit a7de6a02c88bd7950757ea8427235f81b5f9f64d +Subproject commit 55fc2d49c3c7dc94d3324e74635f27a7485e58fa From 493c6782aa378a00c55bbd72faabe0a9252fa48a Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 21 Apr 2023 14:50:22 -0400 Subject: [PATCH 33/92] feat(wasm): bump msgpack-c --- cpp/src/msgpack-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index 55fc2d49c3..7d69ef9b14 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit 55fc2d49c3c7dc94d3324e74635f27a7485e58fa +Subproject commit 7d69ef9b14f6a54f2ae876b584d4001b01683a90 From b35018fe5d164b49b6311e7a413ab490f8867997 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 22 Apr 2023 18:17:42 -0400 Subject: [PATCH 34/92] feat(msgpack): first 'complex' object bound --- cpp/src/barretenberg/common/msgpack_impl.hpp | 80 ------------------- cpp/src/barretenberg/common/serialize.hpp | 3 +- cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp | 3 +- .../dsl/acir_format/logic_constraint.hpp | 3 +- .../dsl/acir_format/sha256_constraint.hpp | 1 + .../ecc/groups/affine_element.hpp | 5 ++ cpp/src/barretenberg/ecc/groups/element.hpp | 4 + .../barretenberg/msgpack/msgpack_concepts.hpp | 9 +++ cpp/src/barretenberg/msgpack/msgpack_impl.hpp | 5 ++ .../barretenberg/msgpack/msgpack_nvp_macro.h | 26 ++++++ .../msgpack/msgpack_struct_array_impl.hpp | 32 ++++++++ .../msgpack/msgpack_struct_map_impl.hpp | 31 +++++++ .../msgpack/msgpack_variant_impl.hpp | 16 ++++ .../plonk/proof_system/types/proof.hpp | 6 ++ .../verification_key/verification_key.hpp | 42 ++++++++++ .../native_aggregation_state.hpp | 3 + cpp/src/msgpack-c | 2 +- 17 files changed, 186 insertions(+), 85 deletions(-) delete mode 100644 cpp/src/barretenberg/common/msgpack_impl.hpp create mode 100644 cpp/src/barretenberg/msgpack/msgpack_concepts.hpp create mode 100644 cpp/src/barretenberg/msgpack/msgpack_impl.hpp create mode 100644 cpp/src/barretenberg/msgpack/msgpack_nvp_macro.h create mode 100644 cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp create mode 100644 cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp create mode 100644 cpp/src/barretenberg/msgpack/msgpack_variant_impl.hpp diff --git a/cpp/src/barretenberg/common/msgpack_impl.hpp b/cpp/src/barretenberg/common/msgpack_impl.hpp deleted file mode 100644 index 3f1db7e49d..0000000000 --- a/cpp/src/barretenberg/common/msgpack_impl.hpp +++ /dev/null @@ -1,80 +0,0 @@ -#pragma once -// Note: heavy header due to serialization logic, don't include if auto parameters will do -#include -#include -#include -#include -#include -#include "throw_or_abort.hpp" -#define MSGPACK_NO_BOOST -#include - -// A simple name-value pair helper for msgpack serialization -#define NVP(x) #x, x - -// User defined class template specialization -namespace msgpack::adaptor { -template -concept AnyType = true; - -struct DefineMapArchive { - template auto operator()(Args&&... args) const - { - return msgpack::type::define_map{ std::forward(args)... }; - } -}; - -struct DefineArchive { - template auto operator()(Args&&... args) const - { - return msgpack::type::define_array{ std::forward(args)... }; - } -}; - -template -concept HasMsgPack = requires(T t, DefineMapArchive ar) { t.msgpack(ar); }; -template -concept HasMsgPackFlat = requires(T t, const DefineArchive& ar) { t.msgpack_flat(ar); }; - -template struct convert { - msgpack::object const& operator()(msgpack::object const& o, T& v) const - { - v.msgpack([&](auto&... args) { DefineMapArchive{}(args...).msgpack_unpack(o); }); - return o; - } -}; - -template struct convert { - msgpack::object const& operator()(msgpack::object const& o, T& v) const - { - v.msgpack_flat([&](auto&... args) { DefineArchive{}(args...).msgpack_unpack(o); }); - return o; - } -}; - -template struct pack { - template packer& operator()(msgpack::packer& o, T const& v) const - { - const_cast(v).msgpack([&](auto&... args) { DefineMapArchive{}(args...).msgpack_pack(o); }); - return o; - } -}; - -template struct pack { - template packer& operator()(msgpack::packer& o, T const& v) const - { - const_cast(v).msgpack_flat([&](auto&... args) { DefineArchive{}(args...).msgpack_pack(o); }); - return o; - } -}; - -template struct pack> { - template - msgpack::packer& operator()(msgpack::packer& o, std::variant const& variant) const - { - std::visit([&o](auto&& arg) { msgpack::pack(o, arg); }, variant); - return o; - } -}; - -} // namespace msgpack::adaptor diff --git a/cpp/src/barretenberg/common/serialize.hpp b/cpp/src/barretenberg/common/serialize.hpp index 2e18b2b5db..f3ef628fea 100644 --- a/cpp/src/barretenberg/common/serialize.hpp +++ b/cpp/src/barretenberg/common/serialize.hpp @@ -40,8 +40,7 @@ #ifndef __i386__ __extension__ using uint128_t = unsigned __int128; #endif -// A simple name-value pair helper for msgpack serialization -#define NVP(x) #x, x + namespace serialize { // Basic integer read / write, to / from raw buffers. // Pointers to buffers are advanced by length of type. diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp index 3ca0d3c3bd..6f8d41a923 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp @@ -3,6 +3,7 @@ #include #include #include "barretenberg/ecc/curves/secp256k1/secp256k1.hpp" +#include "barretenberg/msgpack/msgpack_nvp_macro.h" namespace crypto { namespace ecdsa { @@ -15,7 +16,7 @@ struct signature { std::array r; std::array s; // msgpack entry, update with any new fields - auto msgpack(auto ar) { return ar(NVP(r), NVP(s)); } + auto msgpack(auto ar) { return ar(NVP(r, s)); } }; template diff --git a/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp index b690e4e034..4160ac9a7e 100644 --- a/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp @@ -1,6 +1,7 @@ #pragma once #include #include "barretenberg/stdlib/types/types.hpp" +#include "barretenberg/msgpack/msgpack_nvp_macro.h" namespace acir_format { @@ -14,7 +15,7 @@ struct LogicConstraint { friend bool operator==(LogicConstraint const& lhs, LogicConstraint const& rhs) = default; // msgpack entry, update with any new fields - auto msgpack(auto ar) { return ar(NVP(a), NVP(b), NVP(result), NVP(num_bits), NVP(is_xor_gate)); } + auto msgpack(auto ar) { return ar(NVP(a, b, result, num_bits, is_xor_gate)); } }; void create_logic_gate(plonk::stdlib::types::Composer& composer, diff --git a/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp index a6a5793cba..2528263d74 100644 --- a/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp @@ -2,6 +2,7 @@ #include #include #include "barretenberg/stdlib/types/types.hpp" +#include "barretenberg/msgpack/msgpack_nvp_macro.h" namespace acir_format { diff --git a/cpp/src/barretenberg/ecc/groups/affine_element.hpp b/cpp/src/barretenberg/ecc/groups/affine_element.hpp index 81e6099030..62a4101a25 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element.hpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element.hpp @@ -3,6 +3,7 @@ #include #include #include "barretenberg/ecc/curves/bn254/fq2.hpp" +#include "barretenberg/msgpack/msgpack_nvp_macro.h" namespace barretenberg { namespace group_elements { @@ -63,6 +64,10 @@ template class alignas(64) affine_el constexpr bool operator>(const affine_element& other) const noexcept; constexpr bool operator<(const affine_element& other) const noexcept { return (other > *this); } + // for serialization: update up with new fields + void msgpack(auto ar) { + ar(NVP(x, y)); + } /** * @brief Serialize the point to the given buffer * diff --git a/cpp/src/barretenberg/ecc/groups/element.hpp b/cpp/src/barretenberg/ecc/groups/element.hpp index 1b6096f5ce..0486f63675 100644 --- a/cpp/src/barretenberg/ecc/groups/element.hpp +++ b/cpp/src/barretenberg/ecc/groups/element.hpp @@ -123,6 +123,10 @@ template class alignas(32) element { // } // return { x, y, Fq::one() }; // } + // for serialization: update up with new fields + auto msgpack(auto ar) { + ar(NVP(x, y, z)); + } static void conditional_negate_affine(const affine_element& in, affine_element& out, diff --git a/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp b/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp new file mode 100644 index 0000000000..e57b9bd025 --- /dev/null +++ b/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp @@ -0,0 +1,9 @@ +#pragma once +namespace msgpack { +template +concept HasMsgPack = requires(T t) { t.msgpack([](auto...){}); }; +template +concept HasMsgPackFlat = requires(T t) { t.msgpack_flat([](auto...){}); }; +template +concept SharedPtr = std::is_same_v>; +} \ No newline at end of file diff --git a/cpp/src/barretenberg/msgpack/msgpack_impl.hpp b/cpp/src/barretenberg/msgpack/msgpack_impl.hpp new file mode 100644 index 0000000000..9f25c8f839 --- /dev/null +++ b/cpp/src/barretenberg/msgpack/msgpack_impl.hpp @@ -0,0 +1,5 @@ +#pragma once +// Note: Meant to only be included in compilation units that need msgpack +#include "msgpack_struct_array_impl.hpp" +#include "msgpack_struct_map_impl.hpp" +#include "msgpack_variant_impl.hpp" \ No newline at end of file diff --git a/cpp/src/barretenberg/msgpack/msgpack_nvp_macro.h b/cpp/src/barretenberg/msgpack/msgpack_nvp_macro.h new file mode 100644 index 0000000000..ced3ddc296 --- /dev/null +++ b/cpp/src/barretenberg/msgpack/msgpack_nvp_macro.h @@ -0,0 +1,26 @@ +#pragma once +// NVP: expands to key-value pairs, e.g. NVP(x,y,z) -> "x", x, "y", y", "z", z +// used in msgpack serialization and schema printing + +// hacky counting of variadic macro params +#define VA_NARGS_IMPL(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N +#define VA_NARGS(...) VA_NARGS_IMPL(__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1) + +// name-value pair expansion for variables +// used in msgpack map expansion +// n<=3 case +#define _NVP1(x) #x, x +#define _NVP2(x, y) #x, x, #y, y +#define _NVP3(x, y, z) #x, x, #y, y, #z, z +// n>3 cases +#define _NVP4(x, ...) _NVP1(x), _NVP3(__VA_ARGS__) +#define _NVP5(x, ...) _NVP1(x), _NVP4(__VA_ARGS__) +#define _NVP6(x, ...) _NVP1(x), _NVP5(__VA_ARGS__) +#define _NVP7(x, ...) _NVP1(x), _NVP6(__VA_ARGS__) +#define _NVP8(x, ...) _NVP1(x), _NVP7(__VA_ARGS__) +#define _NVP9(x, ...) _NVP1(x), _NVP8(__VA_ARGS__) +#define _NVP10(x, ...) _NVP(x), _NVP9(__VA_ARGS__) + +#define CONCAT(a, b) a ## b +#define _NVP_N(n) CONCAT(_NVP, n) +#define NVP(...) _NVP_N(VA_NARGS(__VA_ARGS__))(__VA_ARGS__) \ No newline at end of file diff --git a/cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp b/cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp new file mode 100644 index 0000000000..d1f6a62909 --- /dev/null +++ b/cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp @@ -0,0 +1,32 @@ +#pragma once +// Note: Meant to only be included in compilation units that need msgpack +#include +#include +#include +#include +#include +#include "barretenberg/common/throw_or_abort.hpp" +#define MSGPACK_NO_BOOST +#include +#include "msgpack_concepts.hpp" + +namespace msgpack::adaptor { +// converts structs with msgpack_flat() method from a JSON-like dictionary +template struct convert { + msgpack::object const& operator()(msgpack::object const& o, T& v) const + { + v.msgpack_flat([&](auto&... args) { msgpack::type::define_array{args...}.msgpack_unpack(o); }); + return o; + } +}; + +// converts structs with msgpack_flat() method to a JSON-like dictionary +template struct pack { + template packer& operator()(msgpack::packer& o, T const& v) const + { + const_cast(v).msgpack_flat([&](auto&... args) { msgpack::type::define_array{args...}.msgpack_pack(o); }); + return o; + } +}; + +} // namespace msgpack::adaptor diff --git a/cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp b/cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp new file mode 100644 index 0000000000..13a9822e19 --- /dev/null +++ b/cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp @@ -0,0 +1,31 @@ +#pragma once +// Note: heavy header due to serialization logic, don't include if auto parameters will do +#include +#include +#include +#include +#include +#define MSGPACK_NO_BOOST +#include +#include "msgpack_concepts.hpp" + +namespace msgpack::adaptor { +// reads structs with msgpack() method from a JSON-like dictionary +template struct convert { + msgpack::object const& operator()(msgpack::object const& o, T& v) const + { + v.msgpack([&](auto&... args) { msgpack::type::define_map{args...}.msgpack_unpack(o); }); + return o; + } +}; + +// converts structs with msgpack() method from a JSON-like dictionary +template struct pack { + template packer& operator()(msgpack::packer& o, T const& v) const + { + const_cast(v).msgpack([&](auto&... args) { msgpack::type::define_map{args...}.msgpack_pack(o); }); + return o; + } +}; + +} // namespace msgpack::adaptor diff --git a/cpp/src/barretenberg/msgpack/msgpack_variant_impl.hpp b/cpp/src/barretenberg/msgpack/msgpack_variant_impl.hpp new file mode 100644 index 0000000000..24f36dcc58 --- /dev/null +++ b/cpp/src/barretenberg/msgpack/msgpack_variant_impl.hpp @@ -0,0 +1,16 @@ +#pragma once +// Note: Meant to only be included in compilation units that need msgpack + +#include +#include + +namespace msgpack::adaptor { +// writes std::variant to msgpack format (TODO should we read std::variant?) +template +struct pack> { + auto &operator()(auto& o, std::variant const &variant) const { + std::visit([&o](auto &&arg) { msgpack::pack(o, arg); }, variant); + return o; + } +}; +} \ No newline at end of file diff --git a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp index aedf516e04..3eb525197c 100644 --- a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp @@ -4,11 +4,17 @@ #include #include #include "barretenberg/common/serialize.hpp" +#include "barretenberg/msgpack/msgpack_nvp_macro.h" namespace proof_system::plonk { struct proof { std::vector proof_data; + // for serialization: update up with new fields + void msgpack(auto ar) { + ar(NVP(proof_data)); + } + // TODO redundant with msgpack bool operator==(proof const& other) const = default; }; diff --git a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp index 709eea6ee7..efd84c5221 100644 --- a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp @@ -7,6 +7,7 @@ #include "barretenberg/polynomials/evaluation_domain.hpp" #include "barretenberg/crypto/sha256/sha256.hpp" #include "barretenberg/plonk/proof_system/types/polynomial_manifest.hpp" +#include "barretenberg/msgpack/msgpack_concepts.hpp" namespace proof_system::plonk { @@ -18,6 +19,10 @@ struct verification_key_data { bool contains_recursive_proof = false; std::vector recursive_proof_public_input_indices; + // for serialization: update with any new fields + void msgpack(auto ar) { + ar(NVP(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices)); + } barretenberg::fr compress_native(size_t const hash_index = 0); }; @@ -83,6 +88,23 @@ struct verification_key { bool contains_recursive_proof = false; std::vector recursive_proof_public_input_indices; size_t program_width = 3; + // for serialization: update up with new fields + void msgpack_pack(auto& packer) const { + verification_key_data data = { + composer_type, + static_cast(circuit_size), + static_cast(num_public_inputs), + commitments, + contains_recursive_proof, + recursive_proof_public_input_indices + }; + packer.pack(data); + } + void msgpack_unpack(auto obj) { + verification_key_data data = obj; + auto env_crs = std::make_unique(); + *this = verification_key {std::move(data), env_crs->get_verifier_crs()}; + } }; template inline void read(B& buf, verification_key& key) @@ -125,3 +147,23 @@ inline std::ostream& operator<<(std::ostream& os, verification_key const& key) }; } // namespace proof_system::plonk + +// annoyingly, if we want to forward declare anything without including heavy msgpack.hpp headers, we need to +// dig into the internals to do so. +namespace msgpack { +inline namespace v3 { +namespace adaptor { +template struct convert; +// hack: since verification_key has no default constructor, we need to unpack the shared ptr directly +template <> struct convert, void> { + auto const& operator()(auto& obj, std::shared_ptr& verification_key_ptr) const + { + proof_system::plonk::verification_key_data data = obj.convert(); + auto env_crs = std::make_unique(); + verification_key_ptr = std::make_shared(std::move(data), env_crs->get_verifier_crs()); + return obj; + } +}; +} +} +}// namespace msgpack::v3::adaptor \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp b/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp index 47b82b3f57..33eef59c21 100644 --- a/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp +++ b/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp @@ -21,6 +21,9 @@ struct native_aggregation_state { std::vector proof_witness_indices; bool has_data = false; + void msgpack(auto ar) { + ar(NVP(P0, P1, public_inputs, proof_witness_indices, has_data)); + } bool operator==(native_aggregation_state const& other) const { return P0 == other.P0 && P1 == other.P1 && public_inputs == other.public_inputs && diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index 7d69ef9b14..98c4c694d6 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit 7d69ef9b14f6a54f2ae876b584d4001b01683a90 +Subproject commit 98c4c694d65f72f6dfc1d6eae3803ab331aca78e From a2349812b0e8188ab37f4149f5dbbc380031de7f Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 22 Apr 2023 18:50:12 -0400 Subject: [PATCH 35/92] More wasm fixes. Was breaking throw() declaration --- cpp/src/barretenberg/msgpack/msgpack_concepts.hpp | 8 ++++++-- .../barretenberg/msgpack/msgpack_struct_array_impl.hpp | 4 ++-- cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp | 4 ++-- cpp/src/msgpack-c | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp b/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp index e57b9bd025..9649132b35 100644 --- a/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp +++ b/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp @@ -1,9 +1,13 @@ #pragma once + +struct DoNothing { + void operator()(auto...){} +}; namespace msgpack { template -concept HasMsgPack = requires(T t) { t.msgpack([](auto...){}); }; +concept HasMsgPack = requires(T t, DoNothing nop) { t.msgpack(nop); }; template -concept HasMsgPackFlat = requires(T t) { t.msgpack_flat([](auto...){}); }; +concept HasMsgPackFlat = requires(T t, DoNothing nop) { t.msgpack_flat(nop); }; template concept SharedPtr = std::is_same_v>; } \ No newline at end of file diff --git a/cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp b/cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp index d1f6a62909..5ebf93c118 100644 --- a/cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp +++ b/cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp @@ -15,7 +15,7 @@ namespace msgpack::adaptor { template struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const { - v.msgpack_flat([&](auto&... args) { msgpack::type::define_array{args...}.msgpack_unpack(o); }); + v.msgpack_flat([&](auto&... args) { msgpack::type::define_array{args...}.msgpack_unpack(o); }); return o; } }; @@ -24,7 +24,7 @@ template struct convert { template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { - const_cast(v).msgpack_flat([&](auto&... args) { msgpack::type::define_array{args...}.msgpack_pack(o); }); + const_cast(v).msgpack_flat([&](auto&... args) { msgpack::type::define_array{args...}.msgpack_pack(o); }); return o; } }; diff --git a/cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp b/cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp index 13a9822e19..d6d97d1c60 100644 --- a/cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp +++ b/cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp @@ -14,7 +14,7 @@ namespace msgpack::adaptor { template struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const { - v.msgpack([&](auto&... args) { msgpack::type::define_map{args...}.msgpack_unpack(o); }); + v.msgpack([&](auto&... args) { msgpack::type::define_map{args...}.msgpack_unpack(o); }); return o; } }; @@ -23,7 +23,7 @@ template struct convert { template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { - const_cast(v).msgpack([&](auto&... args) { msgpack::type::define_map{args...}.msgpack_pack(o); }); + const_cast(v).msgpack([&](auto&... args) { msgpack::type::define_map{args...}.msgpack_pack(o); }); return o; } }; diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index 98c4c694d6..6c518dde5e 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit 98c4c694d65f72f6dfc1d6eae3803ab331aca78e +Subproject commit 6c518dde5e0334250c69121f70d8af3e34c1eb67 From 403992e3c1c530ba6350f00524e2716ef7df68cf Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 23 Apr 2023 09:38:23 -0400 Subject: [PATCH 36/92] Fix field serialization --- cpp/src/barretenberg/ecc/fields/field_impl.hpp | 8 ++++++-- cpp/src/barretenberg/msgpack/msgpack_concepts.hpp | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cpp/src/barretenberg/ecc/fields/field_impl.hpp b/cpp/src/barretenberg/ecc/fields/field_impl.hpp index ec447d02a9..b734dc2ac6 100644 --- a/cpp/src/barretenberg/ecc/fields/field_impl.hpp +++ b/cpp/src/barretenberg/ecc/fields/field_impl.hpp @@ -612,7 +612,7 @@ template void field::msgpack_pack(auto& packer) const { auto adjusted = from_montgomery_form(); uint64_t bin_data[4] = { - htonll(adjusted.data[0]), htonll(adjusted.data[1]), htonll(adjusted.data[2]), htonll(adjusted.data[3]) + htonll(adjusted.data[3]), htonll(adjusted.data[2]), htonll(adjusted.data[1]), htonll(adjusted.data[0]) }; packer.pack_bin(sizeof(bin_data)); packer.pack_bin_body((const char*)bin_data, sizeof(bin_data)); @@ -621,7 +621,11 @@ template void field::msgpack_pack(auto& packer) const // For serialization template void field::msgpack_unpack(auto o) { - msgpack::read_bin64(o, data, sizeof(data) / sizeof(uint64_t)); + msgpack::read_bin64(o, data, 4); + uint64_t reversed[] = {data[3], data[2], data[1], data[0]}; + for (int i = 0; i < 4; i++) { + data[i] = reversed[i]; + } *this = to_montgomery_form(); } diff --git a/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp b/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp index 9649132b35..1ac24c7ebf 100644 --- a/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp +++ b/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp @@ -7,6 +7,8 @@ namespace msgpack { template concept HasMsgPack = requires(T t, DoNothing nop) { t.msgpack(nop); }; template +concept HasMsgPackPack = requires(T t, DoNothing nop) { t.msgpack_pack(nop); }; +template concept HasMsgPackFlat = requires(T t, DoNothing nop) { t.msgpack_flat(nop); }; template concept SharedPtr = std::is_same_v>; From 489c49204ecfe12543e58c019bf4547af14af574 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 24 Apr 2023 20:06:58 -0400 Subject: [PATCH 37/92] refactoring --- cpp/src/barretenberg/common/msgpack.hpp | 100 ++++++++++++++++++ .../msgpack/concepts.hpp} | 2 - .../msgpack/name_value_pair_macro.hpp} | 11 +- .../msgpack/struct_map_impl.hpp} | 2 +- .../msgpack/variant_impl.hpp} | 2 +- cpp/src/barretenberg/common/msgpack_impl.hpp | 4 + cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp | 6 +- .../dsl/acir_format/logic_constraint.hpp | 6 +- .../dsl/acir_format/sha256_constraint.hpp | 10 +- cpp/src/barretenberg/ecc/fields/.test.cpp | 13 +++ .../barretenberg/ecc/fields/_msgpack_bin.cpp | 24 ----- .../barretenberg/ecc/fields/_msgpack_bin.hpp | 15 --- .../barretenberg/ecc/fields/field_impl.hpp | 6 +- .../ecc/groups/affine_element.hpp | 2 +- cpp/src/barretenberg/msgpack/msgpack_impl.hpp | 5 - .../msgpack/msgpack_struct_array_impl.hpp | 32 ------ .../plonk/proof_system/types/proof.hpp | 2 +- .../verification_key/verification_key.hpp | 41 +++---- .../stdlib/commitment/pedersen/pedersen.cpp | 3 + .../stdlib/primitives/address/address.hpp | 1 + 20 files changed, 167 insertions(+), 120 deletions(-) create mode 100644 cpp/src/barretenberg/common/msgpack.hpp rename cpp/src/barretenberg/{msgpack/msgpack_concepts.hpp => common/msgpack/concepts.hpp} (79%) rename cpp/src/barretenberg/{msgpack/msgpack_nvp_macro.h => common/msgpack/name_value_pair_macro.hpp} (74%) rename cpp/src/barretenberg/{msgpack/msgpack_struct_map_impl.hpp => common/msgpack/struct_map_impl.hpp} (96%) rename cpp/src/barretenberg/{msgpack/msgpack_variant_impl.hpp => common/msgpack/variant_impl.hpp} (94%) create mode 100644 cpp/src/barretenberg/common/msgpack_impl.hpp create mode 100644 cpp/src/barretenberg/ecc/fields/.test.cpp delete mode 100644 cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp delete mode 100644 cpp/src/barretenberg/ecc/fields/_msgpack_bin.hpp delete mode 100644 cpp/src/barretenberg/msgpack/msgpack_impl.hpp delete mode 100644 cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp diff --git a/cpp/src/barretenberg/common/msgpack.hpp b/cpp/src/barretenberg/common/msgpack.hpp new file mode 100644 index 0000000000..fa569730e5 --- /dev/null +++ b/cpp/src/barretenberg/common/msgpack.hpp @@ -0,0 +1,100 @@ +#pragma once +/* Minimal header for msgpack. This should be included, *not* msgpack.hpp directly. +That should be treated as an implementation-only header. + +## Overview + +The Msgpack module allows for efficient serialization and deserialization of data structures. It can be applied to map-like objects, array-like objects, and custom serialization/deserialization logic. + +## Binding objects + +Marking structs/classes with their fields for msgpack allows you to pack and unpack the class. + +1. All objects bound should have a default constructor +2. Objects can be tightly packed as binary (see field_impl.hpp), array-like, or map-like. See below +3. You should list all fields of a class in the below methods, or use the custom method. + +### Map-like Objects + +To make objects serializable as a map-like format, define the `msgpack` method in your class as follows: + +```cpp +void msgpack(auto ar) { + ar(NVP(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices)); +} +``` + +This method maps the object's properties (e.g., `composer_type`, `circuit_size`, etc.) to their respective keys in the serialized data. + +### Array-like Objects + +For array-like objects, define the `msgpack` method similarly: + +```cpp +void msgpack(auto ar) { + ar(NVP(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices)); +} +``` + +This will serialize the object's properties as an array, rather than a map. + +### Custom Serialization and Deserialization + +For custom serialization and deserialization, define `msgpack_pack` and `msgpack_unpack` methods in your class: + +```cpp +// For serialization +template void field::msgpack_pack(auto& packer) const +{ + auto adjusted = from_montgomery_form(); + uint64_t bin_data[4] = { + htonll(adjusted.data[3]), htonll(adjusted.data[2]), htonll(adjusted.data[1]), htonll(adjusted.data[0]) + }; + packer.pack_bin(sizeof(bin_data)); + packer.pack_bin_body((const char*)bin_data, sizeof(bin_data)); +} + +// For deserialization +template void field::msgpack_unpack(auto o) +{ + msgpack::read_bin64(o, data, 4); + uint64_t reversed[] = {data[3], data[2], data[1], data[0]}; + for (int i = 0; i < 4; i++) { + data[i] = reversed[i]; + } + *this = to_montgomery_form(); +} +``` + +These methods allow you to implement custom logic for the serialization and deserialization processes. + + +## Packing/Unpacking + +Only when actually using msgpack to write or read data, include "barretenberg/common/msgpack_impl.hpp". +You can then use msgpack library features to serialize and deserialize C++ objects. + +e.g. packing +``` + // Create a buffer to store the encoded data + msgpack::sbuffer buffer; + msgpack::pack(buffer, obj); + + uint8_t* output = (uint8_t*)aligned_alloc(64, buffer.size()); + memcpy(output, buffer.data(), buffer.size()); + // Convert the buffer data to a string and return it + return { output, buffer.size() }; +``` + +e.g. unpacking + +``` + msgpack::unpack((const char*)encoded_data, encoded_data_size).get().convert(*value); +``` + +*/ +#include "msgpack/name_value_pair_macro.hpp" +#include "msgpack/concepts.hpp" + +// Helper for above documented syntax +#define MSGPACK(...) void msgpack(auto pack_fn) { pack_fn(NVP(__VA_ARGS__)); } \ No newline at end of file diff --git a/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp b/cpp/src/barretenberg/common/msgpack/concepts.hpp similarity index 79% rename from cpp/src/barretenberg/msgpack/msgpack_concepts.hpp rename to cpp/src/barretenberg/common/msgpack/concepts.hpp index 1ac24c7ebf..d340144fd0 100644 --- a/cpp/src/barretenberg/msgpack/msgpack_concepts.hpp +++ b/cpp/src/barretenberg/common/msgpack/concepts.hpp @@ -9,7 +9,5 @@ concept HasMsgPack = requires(T t, DoNothing nop) { t.msgpack(nop); }; template concept HasMsgPackPack = requires(T t, DoNothing nop) { t.msgpack_pack(nop); }; template -concept HasMsgPackFlat = requires(T t, DoNothing nop) { t.msgpack_flat(nop); }; -template concept SharedPtr = std::is_same_v>; } \ No newline at end of file diff --git a/cpp/src/barretenberg/msgpack/msgpack_nvp_macro.h b/cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp similarity index 74% rename from cpp/src/barretenberg/msgpack/msgpack_nvp_macro.h rename to cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp index ced3ddc296..f196282a64 100644 --- a/cpp/src/barretenberg/msgpack/msgpack_nvp_macro.h +++ b/cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp @@ -1,8 +1,10 @@ #pragma once -// NVP: expands to key-value pairs, e.g. NVP(x,y,z) -> "x", x, "y", y", "z", z -// used in msgpack serialization and schema printing -// hacky counting of variadic macro params +/* start of #define NVP: + * expands to name-value pairs (NVPs), e.g. NVP(x,y,z) -> "x", x, "y", y", "z", z + * used in msgpack serialization. */ + +// hacky counting of variadic macro params: #define VA_NARGS_IMPL(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N #define VA_NARGS(...) VA_NARGS_IMPL(__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1) @@ -23,4 +25,5 @@ #define CONCAT(a, b) a ## b #define _NVP_N(n) CONCAT(_NVP, n) -#define NVP(...) _NVP_N(VA_NARGS(__VA_ARGS__))(__VA_ARGS__) \ No newline at end of file +#define NVP(...) _NVP_N(VA_NARGS(__VA_ARGS__))(__VA_ARGS__) +// end of #define NVP \ No newline at end of file diff --git a/cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp similarity index 96% rename from cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp rename to cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp index d6d97d1c60..dab08221da 100644 --- a/cpp/src/barretenberg/msgpack/msgpack_struct_map_impl.hpp +++ b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp @@ -7,7 +7,7 @@ #include #define MSGPACK_NO_BOOST #include -#include "msgpack_concepts.hpp" +#include "concepts.hpp" namespace msgpack::adaptor { // reads structs with msgpack() method from a JSON-like dictionary diff --git a/cpp/src/barretenberg/msgpack/msgpack_variant_impl.hpp b/cpp/src/barretenberg/common/msgpack/variant_impl.hpp similarity index 94% rename from cpp/src/barretenberg/msgpack/msgpack_variant_impl.hpp rename to cpp/src/barretenberg/common/msgpack/variant_impl.hpp index 24f36dcc58..8241ba47ab 100644 --- a/cpp/src/barretenberg/msgpack/msgpack_variant_impl.hpp +++ b/cpp/src/barretenberg/common/msgpack/variant_impl.hpp @@ -1,6 +1,6 @@ #pragma once // Note: Meant to only be included in compilation units that need msgpack - +#define MSGPACK_NO_BOOST #include #include diff --git a/cpp/src/barretenberg/common/msgpack_impl.hpp b/cpp/src/barretenberg/common/msgpack_impl.hpp new file mode 100644 index 0000000000..b2272ec89d --- /dev/null +++ b/cpp/src/barretenberg/common/msgpack_impl.hpp @@ -0,0 +1,4 @@ +#pragma once +// Note: Meant to only be included in compilation units that need msgpack +#include "msgpack/struct_map_impl.hpp" +#include "msgpack/variant_impl.hpp" \ No newline at end of file diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp index 6f8d41a923..f1f738942f 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp @@ -3,7 +3,7 @@ #include #include #include "barretenberg/ecc/curves/secp256k1/secp256k1.hpp" -#include "barretenberg/msgpack/msgpack_nvp_macro.h" +#include "barretenberg/common/msgpack.hpp" namespace crypto { namespace ecdsa { @@ -15,8 +15,8 @@ template struct key_pair { struct signature { std::array r; std::array s; - // msgpack entry, update with any new fields - auto msgpack(auto ar) { return ar(NVP(r, s)); } + // update with any new fields + MSGPACK(r, s); }; template diff --git a/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp index 4160ac9a7e..50f244548a 100644 --- a/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp @@ -1,7 +1,7 @@ #pragma once #include #include "barretenberg/stdlib/types/types.hpp" -#include "barretenberg/msgpack/msgpack_nvp_macro.h" +#include "barretenberg/common/msgpack.hpp" namespace acir_format { @@ -14,8 +14,8 @@ struct LogicConstraint { friend bool operator==(LogicConstraint const& lhs, LogicConstraint const& rhs) = default; - // msgpack entry, update with any new fields - auto msgpack(auto ar) { return ar(NVP(a, b, result, num_bits, is_xor_gate)); } + // for serialization, update with any new fields + MSGPACK(a, b, result, num_bits, is_xor_gate); }; void create_logic_gate(plonk::stdlib::types::Composer& composer, diff --git a/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp index 2528263d74..d345a95f56 100644 --- a/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp @@ -2,7 +2,7 @@ #include #include #include "barretenberg/stdlib/types/types.hpp" -#include "barretenberg/msgpack/msgpack_nvp_macro.h" +#include "barretenberg/common/msgpack.hpp" namespace acir_format { @@ -11,8 +11,8 @@ struct Sha256Input { uint32_t num_bits; friend bool operator==(Sha256Input const& lhs, Sha256Input const& rhs) = default; - // msgpack entry, update with any new fields - auto msgpack(auto ar) { return ar(NVP(witness), NVP(num_bits)); } + // for serialization, update with any new fields + MSGPACK(witness, num_bits); }; struct Sha256Constraint { @@ -20,8 +20,8 @@ struct Sha256Constraint { std::vector result; friend bool operator==(Sha256Constraint const& lhs, Sha256Constraint const& rhs) = default; - // msgpack entry, update with any new fields - auto msgpack(auto ar) { return ar(NVP(inputs), NVP(result)); } + // for serialization, update with any new fields + MSGPACK(inputs, result); }; // This function does not work (properly) because the stdlib:sha256 function is not working correctly for 512 bits diff --git a/cpp/src/barretenberg/ecc/fields/.test.cpp b/cpp/src/barretenberg/ecc/fields/.test.cpp new file mode 100644 index 0000000000..27da6663f2 --- /dev/null +++ b/cpp/src/barretenberg/ecc/fields/.test.cpp @@ -0,0 +1,13 @@ +#include "field.hpp" +#include +#include "barretenberg/common/msgpack_impl.hpp" + +TEST(field_msgpack, basic) +{ + barretenberg::field field(1ull, 2ull, 3ull, 4ull); + barretenberg::field result(0, 0, 0, 0); + msgpack::sbuffer buffer; + msgpack::pack(buffer, field); + msgpack::unpack(buffer.data(), buffer.size()).get().convert(result); + EXPECT_EQ(result, expected); +} diff --git a/cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp b/cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp deleted file mode 100644 index a626ea01ef..0000000000 --- a/cpp/src/barretenberg/ecc/fields/_msgpack_bin.cpp +++ /dev/null @@ -1,24 +0,0 @@ - -#include -#define MSGPACK_NO_BOOST -#include -#include -#include -namespace msgpack { - -void read_bin64(const msgpack::object& obj, uint64_t* data, size_t size) -{ - if (obj.type != msgpack::type::BIN) { - throw_or_abort("Wrong data type when unpacking bin64"); - } - if (obj.via.bin.size != size * sizeof(uint64_t)) { - throw_or_abort("Wrong size data while unpacking bin64"); - } - - uint64_t* bin_data = (uint64_t*)obj.via.bin.ptr; - for (size_t i = 0; i < size; i++) { - data[i] = ntohll(bin_data[i]); - } -} - -} // namespace msgpack diff --git a/cpp/src/barretenberg/ecc/fields/_msgpack_bin.hpp b/cpp/src/barretenberg/ecc/fields/_msgpack_bin.hpp deleted file mode 100644 index 5557516863..0000000000 --- a/cpp/src/barretenberg/ecc/fields/_msgpack_bin.hpp +++ /dev/null @@ -1,15 +0,0 @@ -#include -#include - -// Implementation detail for field serialization -// We are very careful to include msgpack.hpp only in cpp units - -namespace msgpack::v2 { // Hack, have to forward v2 namespace -struct object; -} - -namespace msgpack { - -void read_bin64(const msgpack::v2::object& obj, uint64_t* data, size_t size); - -} diff --git a/cpp/src/barretenberg/ecc/fields/field_impl.hpp b/cpp/src/barretenberg/ecc/fields/field_impl.hpp index b734dc2ac6..c01bc7509b 100644 --- a/cpp/src/barretenberg/ecc/fields/field_impl.hpp +++ b/cpp/src/barretenberg/ecc/fields/field_impl.hpp @@ -12,7 +12,6 @@ #endif #include "field_impl_generic.hpp" -#include "_msgpack_bin.hpp" namespace barretenberg { // template constexpr void field::butterfly(field& left, field& right) noexcept @@ -618,10 +617,11 @@ template void field::msgpack_pack(auto& packer) const packer.pack_bin_body((const char*)bin_data, sizeof(bin_data)); } -// For serialization +// For deserialization template void field::msgpack_unpack(auto o) { - msgpack::read_bin64(o, data, 4); + // Trigger msgpack specialization for binary + (std::array&)*data = o; uint64_t reversed[] = {data[3], data[2], data[1], data[0]}; for (int i = 0; i < 4; i++) { data[i] = reversed[i]; diff --git a/cpp/src/barretenberg/ecc/groups/affine_element.hpp b/cpp/src/barretenberg/ecc/groups/affine_element.hpp index 62a4101a25..a61a1ab41e 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element.hpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element.hpp @@ -3,7 +3,7 @@ #include #include #include "barretenberg/ecc/curves/bn254/fq2.hpp" -#include "barretenberg/msgpack/msgpack_nvp_macro.h" +#include "barretenberg/common/msgpack.hpp" namespace barretenberg { namespace group_elements { diff --git a/cpp/src/barretenberg/msgpack/msgpack_impl.hpp b/cpp/src/barretenberg/msgpack/msgpack_impl.hpp deleted file mode 100644 index 9f25c8f839..0000000000 --- a/cpp/src/barretenberg/msgpack/msgpack_impl.hpp +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once -// Note: Meant to only be included in compilation units that need msgpack -#include "msgpack_struct_array_impl.hpp" -#include "msgpack_struct_map_impl.hpp" -#include "msgpack_variant_impl.hpp" \ No newline at end of file diff --git a/cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp b/cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp deleted file mode 100644 index 5ebf93c118..0000000000 --- a/cpp/src/barretenberg/msgpack/msgpack_struct_array_impl.hpp +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once -// Note: Meant to only be included in compilation units that need msgpack -#include -#include -#include -#include -#include -#include "barretenberg/common/throw_or_abort.hpp" -#define MSGPACK_NO_BOOST -#include -#include "msgpack_concepts.hpp" - -namespace msgpack::adaptor { -// converts structs with msgpack_flat() method from a JSON-like dictionary -template struct convert { - msgpack::object const& operator()(msgpack::object const& o, T& v) const - { - v.msgpack_flat([&](auto&... args) { msgpack::type::define_array{args...}.msgpack_unpack(o); }); - return o; - } -}; - -// converts structs with msgpack_flat() method to a JSON-like dictionary -template struct pack { - template packer& operator()(msgpack::packer& o, T const& v) const - { - const_cast(v).msgpack_flat([&](auto&... args) { msgpack::type::define_array{args...}.msgpack_pack(o); }); - return o; - } -}; - -} // namespace msgpack::adaptor diff --git a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp index 3eb525197c..e305e8ebb0 100644 --- a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp @@ -4,7 +4,7 @@ #include #include #include "barretenberg/common/serialize.hpp" -#include "barretenberg/msgpack/msgpack_nvp_macro.h" +#include "barretenberg/common/msgpack.hpp" namespace proof_system::plonk { diff --git a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp index efd84c5221..fa8194d44b 100644 --- a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp @@ -7,7 +7,7 @@ #include "barretenberg/polynomials/evaluation_domain.hpp" #include "barretenberg/crypto/sha256/sha256.hpp" #include "barretenberg/plonk/proof_system/types/polynomial_manifest.hpp" -#include "barretenberg/msgpack/msgpack_concepts.hpp" +#include "barretenberg/common/msgpack.hpp" namespace proof_system::plonk { @@ -55,6 +55,7 @@ inline bool operator==(verification_key_data const& lhs, verification_key_data c } struct verification_key { + verification_key() = default; verification_key(verification_key_data&& data, std::shared_ptr const& crs); verification_key(const size_t num_gates, const size_t num_inputs, @@ -148,22 +149,22 @@ inline std::ostream& operator<<(std::ostream& os, verification_key const& key) } // namespace proof_system::plonk -// annoyingly, if we want to forward declare anything without including heavy msgpack.hpp headers, we need to -// dig into the internals to do so. -namespace msgpack { -inline namespace v3 { -namespace adaptor { -template struct convert; -// hack: since verification_key has no default constructor, we need to unpack the shared ptr directly -template <> struct convert, void> { - auto const& operator()(auto& obj, std::shared_ptr& verification_key_ptr) const - { - proof_system::plonk::verification_key_data data = obj.convert(); - auto env_crs = std::make_unique(); - verification_key_ptr = std::make_shared(std::move(data), env_crs->get_verifier_crs()); - return obj; - } -}; -} -} -}// namespace msgpack::v3::adaptor \ No newline at end of file +//// annoyingly, if we want to forward declare anything without including heavy msgpack.hpp headers, we need to +//// dig into the internals to do so. +//namespace msgpack { +//inline namespace v3 { +//namespace adaptor { +//template struct convert; +//// hack: since verification_key has no default constructor, we need to unpack the shared ptr directly +//template <> struct convert, void> { +// auto const& operator()(auto& obj, std::shared_ptr& verification_key_ptr) const +// { +// proof_system::plonk::verification_key_data data = obj.convert(); +// auto env_crs = std::make_unique(); +// verification_key_ptr = std::make_shared(std::move(data), env_crs->get_verifier_crs()); +// return obj; +// } +//}; +//} +//} +//}// namespace msgpack::v3::adaptor \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/commitment/pedersen/pedersen.cpp b/cpp/src/barretenberg/stdlib/commitment/pedersen/pedersen.cpp index 57dbc02e62..d790e0a682 100644 --- a/cpp/src/barretenberg/stdlib/commitment/pedersen/pedersen.cpp +++ b/cpp/src/barretenberg/stdlib/commitment/pedersen/pedersen.cpp @@ -78,6 +78,9 @@ field_t pedersen_commitment::compress_unsafe(const field_t& in_left, return pedersen_plookup_commitment::compress({ in_left, in_right }); } + // Make sure generators exist for pedersen (will do nothing if already initialized) + crypto::generators::init_generator_data(); + std::vector accumulators; generator_index_t index_1 = { hash_index, 0 }; generator_index_t index_2 = { hash_index, 1 }; diff --git a/cpp/src/barretenberg/stdlib/primitives/address/address.hpp b/cpp/src/barretenberg/stdlib/primitives/address/address.hpp index d187b96de8..c4cfd3a83e 100644 --- a/cpp/src/barretenberg/stdlib/primitives/address/address.hpp +++ b/cpp/src/barretenberg/stdlib/primitives/address/address.hpp @@ -48,6 +48,7 @@ class address { fr to_field() const { return address_; } + // delegate serialization to field void msgpack_pack(auto& packer) const { address_.msgpack_pack(packer); } void msgpack_unpack(auto const& o) { address_.msgpack_unpack(o); } }; From 69b73c0fefd97e199781774d82c690dfeb306e40 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 24 Apr 2023 20:09:23 -0400 Subject: [PATCH 38/92] Update CMakeLists.txt --- cpp/src/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index e29bafcd3b..d6b2ac5b51 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -38,7 +38,6 @@ else() message(STATUS "Using optimized assembly for field arithmetic.") endif() -set(MSGPACK_USE_BOOST OFF CACHE BOOL "Disable boost" FORCE) add_subdirectory(barretenberg/common) add_subdirectory(barretenberg/env) add_subdirectory(barretenberg/numeric) From 7e9d3fa393c07c61bfb7f25a7f5b4e0aa2586c5e Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 24 Apr 2023 20:12:49 -0400 Subject: [PATCH 39/92] Remove // TODO redundant with msgpack --- cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp | 2 +- cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp | 2 +- cpp/src/barretenberg/plonk/proof_system/types/proof.hpp | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp index dab08221da..77ab4b0f8d 100644 --- a/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp +++ b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp @@ -1,5 +1,5 @@ #pragma once -// Note: heavy header due to serialization logic, don't include if auto parameters will do +// Note: heavy header due to serialization logic, don't include if msgpack.hpp will do #include #include #include diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp index f1f738942f..91e9d7bf48 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp @@ -15,7 +15,7 @@ template struct key_pair { struct signature { std::array r; std::array s; - // update with any new fields + // for serialization, update with any new fields MSGPACK(r, s); }; diff --git a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp index e305e8ebb0..8b4eb1fe6c 100644 --- a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp @@ -14,7 +14,6 @@ struct proof { void msgpack(auto ar) { ar(NVP(proof_data)); } - // TODO redundant with msgpack bool operator==(proof const& other) const = default; }; From 1d350c9c85d49a70e217c87c426352d837ced943 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 26 Apr 2023 07:33:49 -0400 Subject: [PATCH 40/92] Refactor to use macro --- .../ecc/groups/affine_element.hpp | 5 ++- cpp/src/barretenberg/ecc/groups/element.hpp | 4 +-- .../plonk/proof_system/types/proof.hpp | 4 +-- .../verification_key/verification_key.hpp | 33 ++++++------------- .../native_aggregation_state.hpp | 4 +-- 5 files changed, 15 insertions(+), 35 deletions(-) diff --git a/cpp/src/barretenberg/ecc/groups/affine_element.hpp b/cpp/src/barretenberg/ecc/groups/affine_element.hpp index a61a1ab41e..211179b806 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element.hpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element.hpp @@ -65,9 +65,8 @@ template class alignas(64) affine_el constexpr bool operator<(const affine_element& other) const noexcept { return (other > *this); } // for serialization: update up with new fields - void msgpack(auto ar) { - ar(NVP(x, y)); - } + MSGPACK(x, y); + /** * @brief Serialize the point to the given buffer * diff --git a/cpp/src/barretenberg/ecc/groups/element.hpp b/cpp/src/barretenberg/ecc/groups/element.hpp index 0486f63675..a8845f881d 100644 --- a/cpp/src/barretenberg/ecc/groups/element.hpp +++ b/cpp/src/barretenberg/ecc/groups/element.hpp @@ -124,9 +124,7 @@ template class alignas(32) element { // return { x, y, Fq::one() }; // } // for serialization: update up with new fields - auto msgpack(auto ar) { - ar(NVP(x, y, z)); - } + MSGPACK(x, y, z); static void conditional_negate_affine(const affine_element& in, affine_element& out, diff --git a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp index 8b4eb1fe6c..b48262ee86 100644 --- a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp @@ -11,9 +11,7 @@ namespace proof_system::plonk { struct proof { std::vector proof_data; // for serialization: update up with new fields - void msgpack(auto ar) { - ar(NVP(proof_data)); - } + MSGPACK(proof_data); bool operator==(proof const& other) const = default; }; diff --git a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp index fa8194d44b..8c2cf050dc 100644 --- a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp @@ -20,9 +20,14 @@ struct verification_key_data { std::vector recursive_proof_public_input_indices; // for serialization: update with any new fields - void msgpack(auto ar) { - ar(NVP(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices)); - } + MSGPACK( + composer_type, + circuit_size, + num_public_inputs, + commitments, + contains_recursive_proof, + recursive_proof_public_input_indices + ); barretenberg::fr compress_native(size_t const hash_index = 0); }; @@ -55,6 +60,7 @@ inline bool operator==(verification_key_data const& lhs, verification_key_data c } struct verification_key { + // default constructor needed for msgpack unpack verification_key() = default; verification_key(verification_key_data&& data, std::shared_ptr const& crs); verification_key(const size_t num_gates, @@ -89,6 +95,7 @@ struct verification_key { bool contains_recursive_proof = false; std::vector recursive_proof_public_input_indices; size_t program_width = 3; + // for serialization: update up with new fields void msgpack_pack(auto& packer) const { verification_key_data data = { @@ -148,23 +155,3 @@ inline std::ostream& operator<<(std::ostream& os, verification_key const& key) }; } // namespace proof_system::plonk - -//// annoyingly, if we want to forward declare anything without including heavy msgpack.hpp headers, we need to -//// dig into the internals to do so. -//namespace msgpack { -//inline namespace v3 { -//namespace adaptor { -//template struct convert; -//// hack: since verification_key has no default constructor, we need to unpack the shared ptr directly -//template <> struct convert, void> { -// auto const& operator()(auto& obj, std::shared_ptr& verification_key_ptr) const -// { -// proof_system::plonk::verification_key_data data = obj.convert(); -// auto env_crs = std::make_unique(); -// verification_key_ptr = std::make_shared(std::move(data), env_crs->get_verifier_crs()); -// return obj; -// } -//}; -//} -//} -//}// namespace msgpack::v3::adaptor \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp b/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp index 33eef59c21..f91dbf6040 100644 --- a/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp +++ b/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp @@ -21,9 +21,7 @@ struct native_aggregation_state { std::vector proof_witness_indices; bool has_data = false; - void msgpack(auto ar) { - ar(NVP(P0, P1, public_inputs, proof_witness_indices, has_data)); - } + MSGPACK(P0, P1, public_inputs, proof_witness_indices, has_data); bool operator==(native_aggregation_state const& other) const { return P0 == other.P0 && P1 == other.P1 && public_inputs == other.public_inputs && From bf1630f876f17053a5e165d83b226b013f0e125e Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 26 Apr 2023 07:34:23 -0400 Subject: [PATCH 41/92] Refactor to use macro --- cpp/src/msgpack-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index 6c518dde5e..443deb14e0 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit 6c518dde5e0334250c69121f70d8af3e34c1eb67 +Subproject commit 443deb14e00a283612006e187dc4375e53371788 From f903ab13fdceed46e6efe55f04e7145bd83576eb Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 28 Apr 2023 10:14:48 -0400 Subject: [PATCH 42/92] fix printing bug --- .../common/msgpack/name_value_pair_macro.hpp | 11 ++++++++--- .../barretenberg/crypto/generators/generator_data.cpp | 3 ++- cpp/src/barretenberg/ecc/groups/affine_element.hpp | 6 +++--- .../barretenberg/plonk/proof_system/types/proof.hpp | 8 ++++++-- cpp/src/msgpack-c | 2 +- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp b/cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp index f196282a64..83010889f8 100644 --- a/cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp +++ b/cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp @@ -5,8 +5,8 @@ * used in msgpack serialization. */ // hacky counting of variadic macro params: -#define VA_NARGS_IMPL(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N -#define VA_NARGS(...) VA_NARGS_IMPL(__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1) +#define VA_NARGS_IMPL(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, N, ...) N +#define VA_NARGS(...) VA_NARGS_IMPL(__VA_ARGS__, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1) // name-value pair expansion for variables // used in msgpack map expansion @@ -21,7 +21,12 @@ #define _NVP7(x, ...) _NVP1(x), _NVP6(__VA_ARGS__) #define _NVP8(x, ...) _NVP1(x), _NVP7(__VA_ARGS__) #define _NVP9(x, ...) _NVP1(x), _NVP8(__VA_ARGS__) -#define _NVP10(x, ...) _NVP(x), _NVP9(__VA_ARGS__) +#define _NVP10(x, ...) _NVP1(x), _NVP9(__VA_ARGS__) +#define _NVP11(x, ...) _NVP1(x), _NVP10(__VA_ARGS__) +#define _NVP12(x, ...) _NVP1(x), _NVP11(__VA_ARGS__) +#define _NVP13(x, ...) _NVP1(x), _NVP12(__VA_ARGS__) +#define _NVP14(x, ...) _NVP1(x), _NVP13(__VA_ARGS__) +#define _NVP15(x, ...) _NVP1(x), _NVP14(__VA_ARGS__) #define CONCAT(a, b) a ## b #define _NVP_N(n) CONCAT(_NVP, n) diff --git a/cpp/src/barretenberg/crypto/generators/generator_data.cpp b/cpp/src/barretenberg/crypto/generators/generator_data.cpp index bdb9a76479..242be73a4d 100644 --- a/cpp/src/barretenberg/crypto/generators/generator_data.cpp +++ b/cpp/src/barretenberg/crypto/generators/generator_data.cpp @@ -1,5 +1,5 @@ #include "./generator_data.hpp" - +#include namespace crypto { namespace generators { namespace { @@ -207,6 +207,7 @@ std::vector> const& init_generator_data() if (inited && (global_generator_data.size() == size_of_generator_data_array)) { return global_generator_data; } + std::vector generators; std::vector aux_generators; std::vector skew_generators; diff --git a/cpp/src/barretenberg/ecc/groups/affine_element.hpp b/cpp/src/barretenberg/ecc/groups/affine_element.hpp index 211179b806..9999605063 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element.hpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element.hpp @@ -64,9 +64,6 @@ template class alignas(64) affine_el constexpr bool operator>(const affine_element& other) const noexcept; constexpr bool operator<(const affine_element& other) const noexcept { return (other > *this); } - // for serialization: update up with new fields - MSGPACK(x, y); - /** * @brief Serialize the point to the given buffer * @@ -160,6 +157,9 @@ template class alignas(64) affine_el } Fq x; Fq y; + // for serialization: update up with new fields + MSGPACK(x, y); + }; template void read(B& it, affine_element& value) diff --git a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp index b48262ee86..e30014602f 100644 --- a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp @@ -10,8 +10,12 @@ namespace proof_system::plonk { struct proof { std::vector proof_data; - // for serialization: update up with new fields - MSGPACK(proof_data); + void msgpack_pack(auto& packer) const { + packer.pack(proof_data); + } + void msgpack_unpack(auto object) { + proof_data = (std::vector)object; + } bool operator==(proof const& other) const = default; }; diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index 443deb14e0..75dd96925b 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit 443deb14e00a283612006e187dc4375e53371788 +Subproject commit 75dd96925bdbfb105f5e9372ef053bc6e0268f6f From 06159235d7ef04a4ebb37f2f9467fb80ac3f52ee Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 28 Apr 2023 15:06:55 -0400 Subject: [PATCH 43/92] fix: fieldd msgpack endianness fix --- cpp/src/barretenberg/common/log.hpp | 27 ++++++++++--------- .../barretenberg/ecc/fields/field_impl.hpp | 8 +++--- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/cpp/src/barretenberg/common/log.hpp b/cpp/src/barretenberg/common/log.hpp index 729ffcb28c..f0ceaf66b2 100644 --- a/cpp/src/barretenberg/common/log.hpp +++ b/cpp/src/barretenberg/common/log.hpp @@ -18,23 +18,26 @@ namespace { -inline void format_chain(std::ostream&) {} - -template void format_chain(std::ostream& os, T const& first) -{ - os << first; -} - -template void format_chain(std::ostream& os, T const& first, Args const&... args) -{ - os << first; - format_chain(os, args...); +/** + * @brief Utility for logging a tuple type. + * + * @tparam Stream - The stream type e.g. std::ostringstream + * @tparam Args - The arguments type list. + * @param stream The stream. + * @param args - The argument list. + * @return Stream& The stream. + */ +template Stream& operator<<(Stream& stream, const std::tuple& args) { + std::apply([&](const auto&... args) { + ((stream << args), ...); + }, args); + return stream; } template std::string format(Args... args) { std::ostringstream os; - format_chain(os, args...); + ((os << args), ...); return os.str(); } diff --git a/cpp/src/barretenberg/ecc/fields/field_impl.hpp b/cpp/src/barretenberg/ecc/fields/field_impl.hpp index c01bc7509b..87dea05856 100644 --- a/cpp/src/barretenberg/ecc/fields/field_impl.hpp +++ b/cpp/src/barretenberg/ecc/fields/field_impl.hpp @@ -621,10 +621,12 @@ template void field::msgpack_pack(auto& packer) const template void field::msgpack_unpack(auto o) { // Trigger msgpack specialization for binary - (std::array&)*data = o; - uint64_t reversed[] = {data[3], data[2], data[1], data[0]}; + std::array raw_data = o; + // With the binary data, we have to read it as big endian uint64_t's, then correct them to the host endianness + uint64_t* cast_data = (uint64_t*)&raw_data[0]; + uint64_t reversed[] = {ntohll(cast_data[3]), ntohll(cast_data[2]), ntohll(cast_data[1]), ntohll(cast_data[0])}; for (int i = 0; i < 4; i++) { - data[i] = reversed[i]; + data[i] = reversed[i]; } *this = to_montgomery_form(); } From c3d90da211a7961e3b9ab9443739e6abbc2e1ebf Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 28 Apr 2023 16:47:45 -0400 Subject: [PATCH 44/92] fix: remove shared ptr reference --- cpp/src/barretenberg/common/msgpack/concepts.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/src/barretenberg/common/msgpack/concepts.hpp b/cpp/src/barretenberg/common/msgpack/concepts.hpp index d340144fd0..f9add286a7 100644 --- a/cpp/src/barretenberg/common/msgpack/concepts.hpp +++ b/cpp/src/barretenberg/common/msgpack/concepts.hpp @@ -8,6 +8,4 @@ template concept HasMsgPack = requires(T t, DoNothing nop) { t.msgpack(nop); }; template concept HasMsgPackPack = requires(T t, DoNothing nop) { t.msgpack_pack(nop); }; -template -concept SharedPtr = std::is_same_v>; } \ No newline at end of file From 9aafc34ef1649e504bd0ccca03c2668f17e26d03 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 28 Apr 2023 18:21:23 -0400 Subject: [PATCH 45/92] doc --- cpp/src/barretenberg/common/msgpack.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/src/barretenberg/common/msgpack.hpp b/cpp/src/barretenberg/common/msgpack.hpp index fa569730e5..2dfb9b146f 100644 --- a/cpp/src/barretenberg/common/msgpack.hpp +++ b/cpp/src/barretenberg/common/msgpack.hpp @@ -22,8 +22,12 @@ To make objects serializable as a map-like format, define the `msgpack` method i void msgpack(auto ar) { ar(NVP(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices)); } +or +MSGPACK(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices); ``` +This approach assumes 1. all members are default constructible 2. you give it all members 3. all members are writable references + This method maps the object's properties (e.g., `composer_type`, `circuit_size`, etc.) to their respective keys in the serialized data. ### Array-like Objects From 716ff159efe9ac16b4b955ee3888e9cd6d8f3bbc Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 29 Apr 2023 10:03:07 -0400 Subject: [PATCH 46/92] Add static checking for MSGPACK usage --- cpp/src/barretenberg/common/msgpack.hpp | 6 +- .../common/msgpack/struct_map_impl.hpp | 55 ++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/cpp/src/barretenberg/common/msgpack.hpp b/cpp/src/barretenberg/common/msgpack.hpp index 2dfb9b146f..de85e1c484 100644 --- a/cpp/src/barretenberg/common/msgpack.hpp +++ b/cpp/src/barretenberg/common/msgpack.hpp @@ -97,8 +97,12 @@ e.g. unpacking ``` */ +#include #include "msgpack/name_value_pair_macro.hpp" #include "msgpack/concepts.hpp" // Helper for above documented syntax -#define MSGPACK(...) void msgpack(auto pack_fn) { pack_fn(NVP(__VA_ARGS__)); } \ No newline at end of file +#define MSGPACK(...) \ + void msgpack(auto pack_fn) { \ + pack_fn(NVP(__VA_ARGS__)); \ + } diff --git a/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp index 77ab4b0f8d..4f228c8275 100644 --- a/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp +++ b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp @@ -9,12 +9,55 @@ #include #include "concepts.hpp" +namespace msgpack { + template auto drop_keys_impl(Tuple&& tuple, std::index_sequence) + { + // Expand 0 to n/2 to 1 to n+1 (increments of 2) + // Use it to filter the tuple + return std::tie(std::get(std::forward(tuple))...); + } + + template auto drop_keys(std::tuple&& tuple) + { + static_assert(sizeof...(Args) % 2 == 0, "Tuple must contain an even number of elements"); + // Compile time sequence of integers from 0 to n/2 + auto compile_time_0_to_n_div_2 = std::make_index_sequence{}; + return drop_keys_impl(tuple, compile_time_0_to_n_div_2); + } +} // namespace msgpack + +namespace msgpack { +// template +// void __static_check_helper(T object) { +// auto checker = [&](auto &... values) { +// // Make sure we can construct this +// T should_not_error{values...}; +// }; +// object.msgpack( +// [&](auto &... keys_and_values) { std::apply(checker, drop_keys(std::tie(keys_and_values...))); }); +// +// } + template + concept MsgpackConstructible = requires(T object, Args... args) { + T{args...}; + }; +} + namespace msgpack::adaptor { // reads structs with msgpack() method from a JSON-like dictionary template struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const { - v.msgpack([&](auto&... args) { msgpack::type::define_map{args...}.msgpack_unpack(o); }); + static_assert(std::is_default_constructible_v, "MSGPACK requires default-constructible types"); + v.msgpack([&](auto&... args) { + auto static_checker = [&](auto&... value_args) { + static_assert(msgpack::MsgpackConstructible, + "MSGPACK requires a constructor that can take the types listed in MSGPACK. " + "Type or arg count mismatch, or member initializer constructor not available."); + }; + std::apply(static_checker, drop_keys(std::tie(args...))); + msgpack::type::define_map{args...}.msgpack_unpack(o); + }); return o; } }; @@ -23,7 +66,15 @@ template struct convert { template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { - const_cast(v).msgpack([&](auto&... args) { msgpack::type::define_map{args...}.msgpack_pack(o); }); + static_assert(std::is_default_constructible_v, "T requires a default-constructor for MSGPACK"); + const_cast(v).msgpack([&](auto&... args) { + auto static_checker = [&](auto &... value_args) { + static_assert(msgpack::MsgpackConstructible, + "T requires a constructor that can take the types listed in MSGPACK. " + "Type or arg count mismatch, or member initializer constructor not available."); + }; + std::apply(static_checker, drop_keys(std::tie(args...))); + msgpack::type::define_map{args...}.msgpack_pack(o); }); return o; } }; From bc792d0983374ce68960b4099b52ef509317df9e Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 29 Apr 2023 11:28:23 -0400 Subject: [PATCH 47/92] Revert log.hpp change --- cpp/src/barretenberg/common/log.hpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/cpp/src/barretenberg/common/log.hpp b/cpp/src/barretenberg/common/log.hpp index f0ceaf66b2..f581645883 100644 --- a/cpp/src/barretenberg/common/log.hpp +++ b/cpp/src/barretenberg/common/log.hpp @@ -18,22 +18,6 @@ namespace { -/** - * @brief Utility for logging a tuple type. - * - * @tparam Stream - The stream type e.g. std::ostringstream - * @tparam Args - The arguments type list. - * @param stream The stream. - * @param args - The argument list. - * @return Stream& The stream. - */ -template Stream& operator<<(Stream& stream, const std::tuple& args) { - std::apply([&](const auto&... args) { - ((stream << args), ...); - }, args); - return stream; -} - template std::string format(Args... args) { std::ostringstream os; From 73c4b6416cb4eb38f9debd10351b10c897f568b6 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 09:50:55 -0400 Subject: [PATCH 48/92] Update struct_map_impl.hpp --- .../barretenberg/common/msgpack/struct_map_impl.hpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp index 4f228c8275..99a4b9cb03 100644 --- a/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp +++ b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp @@ -27,16 +27,6 @@ namespace msgpack { } // namespace msgpack namespace msgpack { -// template -// void __static_check_helper(T object) { -// auto checker = [&](auto &... values) { -// // Make sure we can construct this -// T should_not_error{values...}; -// }; -// object.msgpack( -// [&](auto &... keys_and_values) { std::apply(checker, drop_keys(std::tie(keys_and_values...))); }); -// -// } template concept MsgpackConstructible = requires(T object, Args... args) { T{args...}; From 276a9a5a867a44ffa32a991967796511251ff6d8 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 09:59:49 -0400 Subject: [PATCH 49/92] Revert --- .../barretenberg/crypto/pedersen_hash/pedersen_lookup.cpp | 6 +----- cpp/src/msgpack-c | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/cpp/src/barretenberg/crypto/pedersen_hash/pedersen_lookup.cpp b/cpp/src/barretenberg/crypto/pedersen_hash/pedersen_lookup.cpp index e5a6fd9884..f455c684be 100644 --- a/cpp/src/barretenberg/crypto/pedersen_hash/pedersen_lookup.cpp +++ b/cpp/src/barretenberg/crypto/pedersen_hash/pedersen_lookup.cpp @@ -66,11 +66,7 @@ void init() { ASSERT(BITS_PER_TABLE < BITS_OF_BETA); ASSERT(BITS_PER_TABLE + BITS_OF_BETA < BITS_ON_CURVE); - bool pedersen_tables_filled = pedersen_iv_table.size() > 0; - for (size_t i = 0; i < NUM_PEDERSEN_TABLES; i++) { - pedersen_tables_filled = (pedersen_tables_filled && (pedersen_tables[i].size() > 0)); - } - if (inited && pedersen_tables_filled) { + if (inited) { return; } generators = grumpkin::g1::derive_generators(); diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index 75dd96925b..a7de6a02c8 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit 75dd96925bdbfb105f5e9372ef053bc6e0268f6f +Subproject commit a7de6a02c88bd7950757ea8427235f81b5f9f64d From 91cb331253a113e7e659cdc78f5769ebe7c5852d Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 09:49:52 -0400 Subject: [PATCH 50/92] remote_build fix --- .circleci/remote_build/remote_build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/remote_build/remote_build b/.circleci/remote_build/remote_build index 92fe12a6af..f8c7b3c806 100755 --- a/.circleci/remote_build/remote_build +++ b/.circleci/remote_build/remote_build @@ -15,6 +15,7 @@ git config remote.origin.promisor true git config remote.origin.partialclonefilter blob:none git fetch --depth 50 origin $COMMIT_HASH git checkout FETCH_HEAD +git submodule update --init echo "Git operations completed..." @@ -24,4 +25,4 @@ echo "Calling setup env..." source ./setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" cd .. echo "Calling build..." -build $@ \ No newline at end of file +build $@ From 50551e63b84de975475b27fd1b5a67215dc5f878 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 09:56:53 -0400 Subject: [PATCH 51/92] Keep trying to init submodules --- .circleci/cond_spot_run_build | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/cond_spot_run_build b/.circleci/cond_spot_run_build index d4fa4723ea..21fef601a6 100755 --- a/.circleci/cond_spot_run_build +++ b/.circleci/cond_spot_run_build @@ -7,6 +7,8 @@ SPEC=$1 shift DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) +init_submodules $REPOSITORY + cd $(query_manifest buildDir $REPOSITORY) LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY) From 3dbe44687ffcd0850af5e9e3ba394ff37aa67bed Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 09:57:43 -0400 Subject: [PATCH 52/92] Keep trying to init submodules --- .circleci/build | 2 ++ .circleci/cond_spot_run_tests | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.circleci/build b/.circleci/build index 3db9f403af..83c573dadd 100755 --- a/.circleci/build +++ b/.circleci/build @@ -20,6 +20,8 @@ echo "Repository: $REPOSITORY" echo "Working directory: $PWD" echo "Dockerfile: $DOCKERFILE" +init_submodules $REPOSITORY + cd $(query_manifest buildDir $REPOSITORY) function fetch_image() { diff --git a/.circleci/cond_spot_run_tests b/.circleci/cond_spot_run_tests index f214624885..f6bf18417f 100755 --- a/.circleci/cond_spot_run_tests +++ b/.circleci/cond_spot_run_tests @@ -5,6 +5,8 @@ set -o pipefail REPOSITORY=$1 shift +init_submodules $REPOSITORY + cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs From 9ca6b534af5129051256b0a7c80dedf0daef2c31 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 10:01:40 -0400 Subject: [PATCH 53/92] Bump --- cpp/src/msgpack-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index a7de6a02c8..75dd96925b 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit a7de6a02c88bd7950757ea8427235f81b5f9f64d +Subproject commit 75dd96925bdbfb105f5e9372ef053bc6e0268f6f From 76ae948a22ed13b4365c3e270998b7aba9695f43 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 10:08:05 -0400 Subject: [PATCH 54/92] Add missing init_submodules --- .circleci/init_submodules | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 .circleci/init_submodules diff --git a/.circleci/init_submodules b/.circleci/init_submodules new file mode 100755 index 0000000000..939c8e4730 --- /dev/null +++ b/.circleci/init_submodules @@ -0,0 +1,18 @@ +#!/bin/bash +# For a given repository, init any required submodules. +set -euo pipefail + +REPOSITORY=$1 + +BUILD_DIR=$(query_manifest buildDir $REPOSITORY) +if [ -d "$BUILD_DIR" ]; then + echo "Initialising any submodules under $REPOSITORY buildDir: $BUILD_DIR" + git submodule update --init --recursive $BUILD_DIR && exit || echo "No submodules under buildDir" +fi + +SUBMODULE_PATH=$(query_manifest submodulePath $REPOSITORY) +if [ -n "$SUBMODULE_PATH" ]; then + # TODO: Needs to actually init all dependency submodules as well. + echo "Initialising submodule: $SUBMODULE_PATH" + git submodule update --init --recursive $SUBMODULE_PATH +fi \ No newline at end of file From 37122e7a519c46bb272d2187721d311614fc0c46 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 10:25:36 -0400 Subject: [PATCH 55/92] Msgpack test fix --- cpp/src/barretenberg/ecc/fields/.test.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/src/barretenberg/ecc/fields/.test.cpp b/cpp/src/barretenberg/ecc/fields/.test.cpp index 27da6663f2..ff01cf4425 100644 --- a/cpp/src/barretenberg/ecc/fields/.test.cpp +++ b/cpp/src/barretenberg/ecc/fields/.test.cpp @@ -1,13 +1,14 @@ #include "field.hpp" #include #include "barretenberg/common/msgpack_impl.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" TEST(field_msgpack, basic) { - barretenberg::field field(1ull, 2ull, 3ull, 4ull); - barretenberg::field result(0, 0, 0, 0); + barretenberg::fr field {1ull, 2ull, 3ull, 4ull}; + barretenberg::fr result {0, 0, 0, 0}; msgpack::sbuffer buffer; msgpack::pack(buffer, field); msgpack::unpack(buffer.data(), buffer.size()).get().convert(result); - EXPECT_EQ(result, expected); + EXPECT_EQ(field, result); } From 66e22e6270754e64e72a03c4e9df19010e67e9a7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 10:32:39 -0400 Subject: [PATCH 56/92] Msgpack test fix --- .../proof_system/polynomial_store/polynomial_store.test.cpp | 2 +- cpp/src/msgpack-c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp index a068cce790..a781ca4442 100644 --- a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp +++ b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp @@ -102,7 +102,7 @@ TEST(PolynomialStore, RangeBasedFor) polynomial_store.put("id_2", std::move(poly2)); // Check that PolynomialStore meets criteria for std::ranges::range - EXPECT_TRUE(std::ranges::range>); + // EXPECT_TRUE(std::ranges::range>); // For example ... Polynomial polynomial_sum(size); diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index 75dd96925b..4c662b80a4 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit 75dd96925bdbfb105f5e9372ef053bc6e0268f6f +Subproject commit 4c662b80a4ae805adf846fa0ceacf92f3f7bd561 From 54b655d00cdb85e23b0c6f62cc3d45c33da8a5f3 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 10:39:48 -0400 Subject: [PATCH 57/92] Msgpack test fix --- cpp/src/msgpack-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index 4c662b80a4..d1160fee87 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit 4c662b80a4ae805adf846fa0ceacf92f3f7bd561 +Subproject commit d1160fee8790173647e031621ee37bc986f5939f From 432ba965e2333a05627c9b3c560d42f2ba8a2b21 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 11:01:51 -0400 Subject: [PATCH 58/92] Msgpack test fix --- cpp/src/msgpack-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index d1160fee87..d0448f5b76 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit d1160fee8790173647e031621ee37bc986f5939f +Subproject commit d0448f5b76dcf9f012be0d73300c3acf65c91adb From ea1b6def1d9266155e5263c6c1e6a5c4529ddcf2 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 1 May 2023 11:11:18 -0400 Subject: [PATCH 59/92] Update polynomial_store.test.cpp --- .../proof_system/polynomial_store/polynomial_store.test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp index a781ca4442..a068cce790 100644 --- a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp +++ b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp @@ -102,7 +102,7 @@ TEST(PolynomialStore, RangeBasedFor) polynomial_store.put("id_2", std::move(poly2)); // Check that PolynomialStore meets criteria for std::ranges::range - // EXPECT_TRUE(std::ranges::range>); + EXPECT_TRUE(std::ranges::range>); // For example ... Polynomial polynomial_sum(size); From 38fb021e1920f91a6665e47487c2bfeb32d91731 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 May 2023 14:38:27 -0400 Subject: [PATCH 60/92] Merge master --- .gitmodules | 3 -- cpp/src/barretenberg/common/msgpack.hpp | 35 ++++++------- .../common/msgpack/struct_map_impl.hpp | 49 ++++++++++--------- .../standard_plonk_composer_helper.cpp | 2 + .../turbo_plonk_composer_helper.cpp | 2 + .../ultra_plonk_composer_helper.cpp | 2 + .../plonk/composer/turbo_composer.cpp | 3 ++ .../plonk/composer/ultra_composer.cpp | 3 ++ .../random_widgets/permutation_widget.hpp | 2 + .../widgets/random_widgets/plookup_widget.hpp | 2 + .../polynomial_store.test.cpp | 2 +- .../solidity_helpers/proof_gen.cpp | 4 ++ .../recursion/verifier/verifier.test.cpp | 3 ++ foundation | 1 - 14 files changed, 64 insertions(+), 49 deletions(-) delete mode 160000 foundation diff --git a/.gitmodules b/.gitmodules index 41828bf09e..0a2414020a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "foundation"] - path = foundation - url = git@github.com:AztecProtocol/foundation.git [submodule "cpp/src/msgpack-c"] path = cpp/src/msgpack-c url = https://github.com/AztecProtocol/msgpack-c diff --git a/cpp/src/barretenberg/common/msgpack.hpp b/cpp/src/barretenberg/common/msgpack.hpp index de85e1c484..2bb3399829 100644 --- a/cpp/src/barretenberg/common/msgpack.hpp +++ b/cpp/src/barretenberg/common/msgpack.hpp @@ -4,7 +4,8 @@ That should be treated as an implementation-only header. ## Overview -The Msgpack module allows for efficient serialization and deserialization of data structures. It can be applied to map-like objects, array-like objects, and custom serialization/deserialization logic. +The Msgpack module allows for efficient serialization and deserialization of data structures. It can be applied to +map-like objects, array-like objects, and custom serialization/deserialization logic. ## Binding objects @@ -14,33 +15,26 @@ Marking structs/classes with their fields for msgpack allows you to pack and unp 2. Objects can be tightly packed as binary (see field_impl.hpp), array-like, or map-like. See below 3. You should list all fields of a class in the below methods, or use the custom method. -### Map-like Objects +### Typical Objects To make objects serializable as a map-like format, define the `msgpack` method in your class as follows: ```cpp void msgpack(auto ar) { - ar(NVP(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices)); + ar(NVP(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, +recursive_proof_public_input_indices)); } or -MSGPACK(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices); +MSGPACK(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, +recursive_proof_public_input_indices); ``` -This approach assumes 1. all members are default constructible 2. you give it all members 3. all members are writable references +This approach assumes 1. all members are default constructible 2. you give it all members 3. all members are writable +references -This method maps the object's properties (e.g., `composer_type`, `circuit_size`, etc.) to their respective keys in the serialized data. +This method maps the object's properties (e.g., `composer_type`, `circuit_size`, etc.) to their respective keys in the +serialized data. -### Array-like Objects - -For array-like objects, define the `msgpack` method similarly: - -```cpp -void msgpack(auto ar) { - ar(NVP(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices)); -} -``` - -This will serialize the object's properties as an array, rather than a map. ### Custom Serialization and Deserialization @@ -102,7 +96,8 @@ e.g. unpacking #include "msgpack/concepts.hpp" // Helper for above documented syntax -#define MSGPACK(...) \ - void msgpack(auto pack_fn) { \ - pack_fn(NVP(__VA_ARGS__)); \ +#define MSGPACK(...) \ + void msgpack(auto pack_fn) \ + { \ + pack_fn(NVP(__VA_ARGS__)); \ } diff --git a/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp index 99a4b9cb03..006b766a1f 100644 --- a/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp +++ b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp @@ -10,27 +10,25 @@ #include "concepts.hpp" namespace msgpack { - template auto drop_keys_impl(Tuple&& tuple, std::index_sequence) - { - // Expand 0 to n/2 to 1 to n+1 (increments of 2) - // Use it to filter the tuple - return std::tie(std::get(std::forward(tuple))...); - } +template auto drop_keys_impl(Tuple&& tuple, std::index_sequence) +{ + // Expand 0 to n/2 to 1 to n+1 (increments of 2) + // Use it to filter the tuple + return std::tie(std::get(std::forward(tuple))...); +} - template auto drop_keys(std::tuple&& tuple) - { - static_assert(sizeof...(Args) % 2 == 0, "Tuple must contain an even number of elements"); - // Compile time sequence of integers from 0 to n/2 - auto compile_time_0_to_n_div_2 = std::make_index_sequence{}; - return drop_keys_impl(tuple, compile_time_0_to_n_div_2); - } +template auto drop_keys(std::tuple&& tuple) +{ + static_assert(sizeof...(Args) % 2 == 0, "Tuple must contain an even number of elements"); + // Compile time sequence of integers from 0 to n/2 + auto compile_time_0_to_n_div_2 = std::make_index_sequence{}; + return drop_keys_impl(tuple, compile_time_0_to_n_div_2); +} } // namespace msgpack namespace msgpack { - template - concept MsgpackConstructible = requires(T object, Args... args) { - T{args...}; - }; +template +concept MsgpackConstructible = requires(T object, Args... args) { T{ args... }; }; } namespace msgpack::adaptor { @@ -38,15 +36,16 @@ namespace msgpack::adaptor { template struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const { - static_assert(std::is_default_constructible_v, "MSGPACK requires default-constructible types"); + static_assert(std::is_default_constructible_v, + "MSGPACK requires default-constructible types (used during unpacking)"); v.msgpack([&](auto&... args) { auto static_checker = [&](auto&... value_args) { static_assert(msgpack::MsgpackConstructible, - "MSGPACK requires a constructor that can take the types listed in MSGPACK. " - "Type or arg count mismatch, or member initializer constructor not available."); + "MSGPACK requires a constructor that can take the types listed in MSGPACK. " + "Type or arg count mismatch, or member initializer constructor not available."); }; std::apply(static_checker, drop_keys(std::tie(args...))); - msgpack::type::define_map{args...}.msgpack_unpack(o); + msgpack::type::define_map{ args... }.msgpack_unpack(o); }); return o; } @@ -56,15 +55,17 @@ template struct convert { template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { - static_assert(std::is_default_constructible_v, "T requires a default-constructor for MSGPACK"); + static_assert(std::is_default_constructible_v, + "MSGPACK requires default-constructible types (used during unpacking)"); const_cast(v).msgpack([&](auto&... args) { - auto static_checker = [&](auto &... value_args) { + auto static_checker = [&](auto&... value_args) { static_assert(msgpack::MsgpackConstructible, "T requires a constructor that can take the types listed in MSGPACK. " "Type or arg count mismatch, or member initializer constructor not available."); }; std::apply(static_checker, drop_keys(std::tie(args...))); - msgpack::type::define_map{args...}.msgpack_pack(o); }); + msgpack::type::define_map{ args... }.msgpack_pack(o); + }); return o; } }; diff --git a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/standard_plonk_composer_helper.cpp b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/standard_plonk_composer_helper.cpp index dabf5e4bc1..b9c350658a 100644 --- a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/standard_plonk_composer_helper.cpp +++ b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/standard_plonk_composer_helper.cpp @@ -8,6 +8,8 @@ #include "barretenberg/proof_system/composer/composer_helper_lib.hpp" #include "barretenberg/plonk/composer/splitting_tmp/composer_helper/composer_helper_lib.hpp" #include "barretenberg/proof_system/composer/permutation_helper.hpp" +// Be explicit about including the heavy implementation headers: +#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" #include #include diff --git a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/turbo_plonk_composer_helper.cpp b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/turbo_plonk_composer_helper.cpp index c367d9ccd1..7a335291df 100644 --- a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/turbo_plonk_composer_helper.cpp +++ b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/turbo_plonk_composer_helper.cpp @@ -13,6 +13,8 @@ #include "barretenberg/proof_system/composer/permutation_helper.hpp" #include "barretenberg/proof_system/composer/composer_helper_lib.hpp" #include "barretenberg/plonk/composer/splitting_tmp/composer_helper/composer_helper_lib.hpp" +// Be explicit about including the heavy implementation headers: +#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" using namespace barretenberg; diff --git a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/ultra_plonk_composer_helper.cpp b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/ultra_plonk_composer_helper.cpp index 6dd56226e1..f5b1714c5b 100644 --- a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/ultra_plonk_composer_helper.cpp +++ b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/ultra_plonk_composer_helper.cpp @@ -5,6 +5,8 @@ #include "barretenberg/proof_system/circuit_constructors/ultra_circuit_constructor.hpp" #include "barretenberg/proof_system/composer/permutation_helper.hpp" #include "barretenberg/plonk/proof_system/commitment_scheme/kate_commitment_scheme.hpp" +// Be explicit about including the heavy implementation headers: +#include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp" #include #include diff --git a/cpp/src/barretenberg/plonk/composer/turbo_composer.cpp b/cpp/src/barretenberg/plonk/composer/turbo_composer.cpp index f3339cf147..bee4a8a90b 100644 --- a/cpp/src/barretenberg/plonk/composer/turbo_composer.cpp +++ b/cpp/src/barretenberg/plonk/composer/turbo_composer.cpp @@ -2,6 +2,9 @@ #include "barretenberg/ecc/curves/bn254/scalar_multiplication/scalar_multiplication.hpp" #include "barretenberg/numeric/bitop/get_msb.hpp" #include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp" +// Be explicit about including the heavy implementation headers: +#include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp" +#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" #include "barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_arithmetic_widget.hpp" #include "barretenberg/plonk/proof_system/widgets/transition_widgets/fixed_base_widget.hpp" #include "barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_logic_widget.hpp" diff --git a/cpp/src/barretenberg/plonk/composer/ultra_composer.cpp b/cpp/src/barretenberg/plonk/composer/ultra_composer.cpp index 1f3e9fb3ed..1eb5a4fcc6 100644 --- a/cpp/src/barretenberg/plonk/composer/ultra_composer.cpp +++ b/cpp/src/barretenberg/plonk/composer/ultra_composer.cpp @@ -11,6 +11,9 @@ #include "barretenberg/plonk/proof_system/widgets/transition_widgets/plookup_auxiliary_widget.hpp" #include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp" #include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget.hpp" +// Be explicit about including the heavy implementation headers: +#include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp" +#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" #include "barretenberg/plonk/proof_system/commitment_scheme/kate_commitment_scheme.hpp" #include "barretenberg/srs/reference_string/file_reference_string.hpp" diff --git a/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp b/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp index 98e58b0bad..6ae47ded0a 100644 --- a/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp @@ -44,4 +44,6 @@ class ProverPermutationWidget : public ProverRandomWidget { } // namespace proof_system::plonk +// The class implementation is expensive to compile +// Include it only in translation (.cpp) units that need it #include "./permutation_widget_impl.hpp" \ No newline at end of file diff --git a/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget.hpp b/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget.hpp index 623f3a6193..b76697e415 100644 --- a/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget.hpp @@ -48,4 +48,6 @@ class ProverPlookupWidget : public ProverRandomWidget { } // namespace proof_system::plonk +// The class implementation is expensive to compile +// Include it only in translation (.cpp) units that need it #include "./plookup_widget_impl.hpp" \ No newline at end of file diff --git a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp index a068cce790..a781ca4442 100644 --- a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp +++ b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp @@ -102,7 +102,7 @@ TEST(PolynomialStore, RangeBasedFor) polynomial_store.put("id_2", std::move(poly2)); // Check that PolynomialStore meets criteria for std::ranges::range - EXPECT_TRUE(std::ranges::range>); + // EXPECT_TRUE(std::ranges::range>); // For example ... Polynomial polynomial_sum(size); diff --git a/cpp/src/barretenberg/solidity_helpers/proof_gen.cpp b/cpp/src/barretenberg/solidity_helpers/proof_gen.cpp index 4765895ec5..4c0368370b 100644 --- a/cpp/src/barretenberg/solidity_helpers/proof_gen.cpp +++ b/cpp/src/barretenberg/solidity_helpers/proof_gen.cpp @@ -5,6 +5,10 @@ #include "barretenberg/plonk/composer/standard_composer.hpp" #include "barretenberg/plonk/composer/ultra_composer.hpp" +// Be explicit about including the heavy implementation headers: +#include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp" +#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" + #include "circuits/blake_circuit.hpp" #include "circuits/add_2_circuit.hpp" #include "circuits/recursive_circuit.hpp" diff --git a/cpp/src/barretenberg/stdlib/recursion/verifier/verifier.test.cpp b/cpp/src/barretenberg/stdlib/recursion/verifier/verifier.test.cpp index 82e8042b02..615112e262 100644 --- a/cpp/src/barretenberg/stdlib/recursion/verifier/verifier.test.cpp +++ b/cpp/src/barretenberg/stdlib/recursion/verifier/verifier.test.cpp @@ -3,6 +3,9 @@ #include "barretenberg/transcript/transcript.hpp" #include "barretenberg/plonk/proof_system/proving_key/serialize.hpp" #include "barretenberg/stdlib/primitives/curves/bn254.hpp" +// Be explicit about including the heavy implementation headers: +#include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp" +#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" #include "barretenberg/ecc/curves/bn254/fq12.hpp" #include "barretenberg/ecc/curves/bn254/pairing.hpp" #include "../../hash/blake3s/blake3s.hpp" diff --git a/foundation b/foundation deleted file mode 160000 index 003b3a640d..0000000000 --- a/foundation +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 003b3a640d9b98a002d6c231ee75cdf0136e8116 From de0292546f150f75561517037e2995705348a64d Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 May 2023 16:40:36 -0400 Subject: [PATCH 61/92] Update msgpack error --- cpp/src/barretenberg/common/msgpack.hpp | 1 - .../common/msgpack/name_value_pair_macro.hpp | 10 ++++++++-- .../barretenberg/common/msgpack/struct_map_impl.hpp | 5 +++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cpp/src/barretenberg/common/msgpack.hpp b/cpp/src/barretenberg/common/msgpack.hpp index 2bb3399829..b93de50281 100644 --- a/cpp/src/barretenberg/common/msgpack.hpp +++ b/cpp/src/barretenberg/common/msgpack.hpp @@ -89,7 +89,6 @@ e.g. unpacking ``` msgpack::unpack((const char*)encoded_data, encoded_data_size).get().convert(*value); ``` - */ #include #include "msgpack/name_value_pair_macro.hpp" diff --git a/cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp b/cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp index 83010889f8..1f3ccc3474 100644 --- a/cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp +++ b/cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp @@ -5,8 +5,9 @@ * used in msgpack serialization. */ // hacky counting of variadic macro params: -#define VA_NARGS_IMPL(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, N, ...) N -#define VA_NARGS(...) VA_NARGS_IMPL(__VA_ARGS__, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1) +#define VA_NARGS_IMPL(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, N, ...) N +// AD: support for 20 fields!? one may ask. Well, after 15 not being enough... +#define VA_NARGS(...) VA_NARGS_IMPL(__VA_ARGS__, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1) // name-value pair expansion for variables // used in msgpack map expansion @@ -27,6 +28,11 @@ #define _NVP13(x, ...) _NVP1(x), _NVP12(__VA_ARGS__) #define _NVP14(x, ...) _NVP1(x), _NVP13(__VA_ARGS__) #define _NVP15(x, ...) _NVP1(x), _NVP14(__VA_ARGS__) +#define _NVP16(x, ...) _NVP1(x), _NVP15(__VA_ARGS__) +#define _NVP17(x, ...) _NVP1(x), _NVP16(__VA_ARGS__) +#define _NVP18(x, ...) _NVP1(x), _NVP17(__VA_ARGS__) +#define _NVP19(x, ...) _NVP1(x), _NVP18(__VA_ARGS__) +#define _NVP20(x, ...) _NVP1(x), _NVP19(__VA_ARGS__) #define CONCAT(a, b) a ## b #define _NVP_N(n) CONCAT(_NVP, n) diff --git a/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp index 006b766a1f..81dd27ab69 100644 --- a/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp +++ b/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp @@ -60,8 +60,9 @@ template struct pack { const_cast(v).msgpack([&](auto&... args) { auto static_checker = [&](auto&... value_args) { static_assert(msgpack::MsgpackConstructible, - "T requires a constructor that can take the types listed in MSGPACK. " - "Type or arg count mismatch, or member initializer constructor not available."); + "T requires a constructor that can take the fields listed in MSGPACK (T will be in template parameters in the compiler stack trace)" + "Check the MSGPACK macro usage in T for incompleteness or wrong order." + "Alternatively, a matching member initializer constructor might not be available for T and should be defined."); }; std::apply(static_checker, drop_keys(std::tie(args...))); msgpack::type::define_map{ args... }.msgpack_pack(o); From 0018e9374adae25203ed318f42b4e72ce28458ee Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 May 2023 19:25:02 -0400 Subject: [PATCH 62/92] Better abort distinguishing --- cpp/src/barretenberg/common/throw_or_abort.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/common/throw_or_abort.hpp b/cpp/src/barretenberg/common/throw_or_abort.hpp index dcaad88433..6232587246 100644 --- a/cpp/src/barretenberg/common/throw_or_abort.hpp +++ b/cpp/src/barretenberg/common/throw_or_abort.hpp @@ -7,7 +7,7 @@ inline void throw_or_abort [[noreturn]] (std::string const& err) #ifndef __wasm__ throw std::runtime_error(err); #else - info(err); + info("abort: ", err); std::abort(); #endif } \ No newline at end of file From 4f94d5533816f7a8cbe8740493e9b0714129331f Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 May 2023 19:25:14 -0400 Subject: [PATCH 63/92] fix: join split VK hash --- .../join_split_example/proofs/join_split/join_split.test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp b/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp index 7081ce5e38..aa7e516609 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp +++ b/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp @@ -808,7 +808,7 @@ TEST_F(join_split_tests, test_0_input_notes_and_detect_circuit_change) constexpr uint32_t CIRCUIT_GATE_COUNT = 185573; constexpr uint32_t GATES_NEXT_POWER_OF_TWO = 524288; - const uint256_t VK_HASH("21389d5392ee23ffc96984689150b63d62113678b1ba127346a0ec72df842354"); + const uint256_t VK_HASH("13eb88883e80efb9bf306af2962cd1a49e9fa1b0bfb2d4b563b95217a17bcc74"); auto number_of_gates_js = result.number_of_gates; auto vk_hash_js = get_verification_key()->sha256_hash(); From 5c569c7a8ba40c9d820f0c497b175ecc0eb3ac2b Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 14 May 2023 15:53:03 -0400 Subject: [PATCH 64/92] Serialization updates --- cpp/src/CMakeLists.txt | 1 + cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp | 2 +- .../barretenberg/crypto/ecdsa/ecdsa.test.cpp | 9 +- .../barretenberg/crypto/ecdsa/ecdsa_impl.hpp | 2 +- cpp/src/barretenberg/crypto/hmac/hmac.hpp | 2 +- .../crypto/pedersen_commitment/c_bind.cpp | 2 +- .../crypto/pedersen_commitment/c_bind.hpp | 2 +- .../crypto/pedersen_hash/c_bind.cpp | 2 +- .../crypto/pedersen_hash/c_bind.hpp | 2 +- .../crypto/schnorr/proof_of_possession.hpp | 2 +- .../barretenberg/crypto/schnorr/schnorr.hpp | 2 +- .../dsl/acir_format/acir_format.test.cpp | 7 +- .../dsl/acir_format/range_constraint.hpp | 2 +- cpp/src/barretenberg/ecc/curves/bn254/fq.hpp | 10 +- .../barretenberg/ecc/curves/bn254/fq.test.cpp | 7 + cpp/src/barretenberg/ecc/curves/bn254/fr.hpp | 10 +- .../barretenberg/ecc/curves/bn254/fr.test.cpp | 7 + cpp/src/barretenberg/ecc/curves/bn254/g1.hpp | 10 +- cpp/src/barretenberg/ecc/fields/.test.cpp | 14 -- cpp/src/barretenberg/ecc/fields/field.hpp | 2 +- .../ecc/groups/affine_element.hpp | 3 +- .../ecc/groups/affine_element.test.cpp | 9 +- cpp/src/barretenberg/ecc/serialize.test.cpp | 9 + .../honk/sumcheck/polynomials/univariate.hpp | 2 +- .../barretenberg/honk/sumcheck/sumcheck.hpp | 2 +- .../honk/transcript/transcript.hpp | 2 +- .../proofs/notes/native/bridge_call_data.hpp | 2 +- .../proofs/notes/native/claim/claim_note.hpp | 2 +- .../notes/native/claim/claim_note_tx_data.hpp | 2 +- .../claim/complete_partial_commitment.hpp | 2 +- .../notes/native/claim/compute_nullifier.hpp | 2 +- .../value/create_partial_commitment.hpp | 2 +- .../proofs/notes/native/value/value_note.hpp | 2 +- .../barretenberg/numeric/uint128/uint128.hpp | 2 +- .../barretenberg/numeric/uint256/uint256.hpp | 2 +- .../plonk/proof_system/prover/c_bind.cpp | 15 +- .../proof_system/proving_key/serialize.hpp | 2 +- .../types/polynomial_manifest.hpp | 2 +- .../plonk/proof_system/types/proof.hpp | 12 +- .../verification_key/verification_key.hpp | 8 +- .../arithmetization/gate_data.hpp | 2 +- cpp/src/barretenberg/serialize/CMakeLists.txt | 7 + cpp/src/barretenberg/serialize/cbind.hpp | 139 ++++++++++++++ cpp/src/barretenberg/serialize/cbind_fwd.h | 10 + .../legacy_serialize.hpp} | 3 +- .../{common => serialize}/msgpack.hpp | 8 +- .../msgpack_impl/check_memory_span.hpp | 76 ++++++++ .../msgpack_impl}/concepts.hpp | 0 .../msgpack_impl}/msgpack_impl.hpp | 4 +- .../msgpack_impl}/name_value_pair_macro.hpp | 0 .../serialize/msgpack_impl/schema_impl.hpp | 176 ++++++++++++++++++ .../serialize/msgpack_impl/schema_name.hpp | 33 ++++ .../msgpack_impl}/struct_map_impl.hpp | 0 .../msgpack_impl}/variant_impl.hpp | 0 .../serialize/msgpack_schema.test.cpp | 86 +++++++++ .../barretenberg/serialize/raw_pointer.hpp | 25 +++ .../barretenberg/serialize/test_helper.hpp | 29 +++ .../stdlib/primitives/address/address.hpp | 10 +- 58 files changed, 726 insertions(+), 63 deletions(-) delete mode 100644 cpp/src/barretenberg/ecc/fields/.test.cpp create mode 100644 cpp/src/barretenberg/ecc/serialize.test.cpp create mode 100644 cpp/src/barretenberg/serialize/CMakeLists.txt create mode 100644 cpp/src/barretenberg/serialize/cbind.hpp create mode 100644 cpp/src/barretenberg/serialize/cbind_fwd.h rename cpp/src/barretenberg/{common/serialize.hpp => serialize/legacy_serialize.hpp} (98%) rename cpp/src/barretenberg/{common => serialize}/msgpack.hpp (93%) create mode 100644 cpp/src/barretenberg/serialize/msgpack_impl/check_memory_span.hpp rename cpp/src/barretenberg/{common/msgpack => serialize/msgpack_impl}/concepts.hpp (100%) rename cpp/src/barretenberg/{common => serialize/msgpack_impl}/msgpack_impl.hpp (54%) rename cpp/src/barretenberg/{common/msgpack => serialize/msgpack_impl}/name_value_pair_macro.hpp (100%) create mode 100644 cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp create mode 100644 cpp/src/barretenberg/serialize/msgpack_impl/schema_name.hpp rename cpp/src/barretenberg/{common/msgpack => serialize/msgpack_impl}/struct_map_impl.hpp (100%) rename cpp/src/barretenberg/{common/msgpack => serialize/msgpack_impl}/variant_impl.hpp (100%) create mode 100644 cpp/src/barretenberg/serialize/msgpack_schema.test.cpp create mode 100644 cpp/src/barretenberg/serialize/raw_pointer.hpp create mode 100644 cpp/src/barretenberg/serialize/test_helper.hpp diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 52dcdc0d91..7f12367c67 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -52,6 +52,7 @@ add_subdirectory(barretenberg/plonk) add_subdirectory(barretenberg/stdlib) add_subdirectory(barretenberg/join_split_example) add_subdirectory(barretenberg/dsl) +add_subdirectory(barretenberg/serialize) add_subdirectory(barretenberg/solidity_helpers) if(BENCHMARKS) diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp index 31e1819008..4ceb5ef6dd 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp @@ -3,7 +3,7 @@ #include #include #include "barretenberg/ecc/curves/secp256k1/secp256k1.hpp" -#include "barretenberg/common/msgpack.hpp" +#include "barretenberg/serialize/msgpack.hpp" namespace crypto { namespace ecdsa { diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp index e45352f053..fa38d0a738 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp @@ -1,11 +1,18 @@ #include "ecdsa.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "barretenberg/ecc/curves/secp256r1/secp256r1.hpp" -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/serialize/test_helper.hpp" #include using namespace barretenberg; +TEST(ecdsa, msgpack) +{ + auto [actual, expected] = msgpack_roundtrip(crypto::ecdsa::signature{{}, {}, {}}); + EXPECT_EQ(actual, expected); +} + TEST(ecdsa, verify_signature_grumpkin_sha256) { std::string message = "The quick brown dog jumped over the lazy fox."; diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa_impl.hpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa_impl.hpp index 33cec9c433..251b350bc1 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa_impl.hpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa_impl.hpp @@ -1,7 +1,7 @@ #pragma once #include "barretenberg/numeric/uint256/uint256.hpp" -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "../hmac/hmac.hpp" namespace crypto { diff --git a/cpp/src/barretenberg/crypto/hmac/hmac.hpp b/cpp/src/barretenberg/crypto/hmac/hmac.hpp index e78ef5206c..6652626779 100644 --- a/cpp/src/barretenberg/crypto/hmac/hmac.hpp +++ b/cpp/src/barretenberg/crypto/hmac/hmac.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include #include #include diff --git a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp index 1b58491c3c..1840e11894 100644 --- a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp +++ b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp @@ -1,7 +1,7 @@ #include "c_bind.hpp" #include "pedersen.hpp" #include "pedersen_lookup.hpp" -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/common/timer.hpp" #include "barretenberg/common/mem.hpp" #include "barretenberg/common/streams.hpp" diff --git a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp index 59b4ef6c40..38f57fe899 100644 --- a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp +++ b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/common/timer.hpp" #include "barretenberg/common/mem.hpp" #include "barretenberg/common/streams.hpp" diff --git a/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp b/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp index ffffc314dc..a3db4cc67c 100644 --- a/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp +++ b/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp @@ -1,7 +1,7 @@ #include "c_bind.hpp" #include "pedersen.hpp" #include "pedersen_lookup.hpp" -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/common/timer.hpp" #include "barretenberg/common/mem.hpp" #include "barretenberg/common/streams.hpp" diff --git a/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp b/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp index cd2804b26c..b7df93f0a4 100644 --- a/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp +++ b/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/common/timer.hpp" #include "barretenberg/common/mem.hpp" #include "barretenberg/common/streams.hpp" diff --git a/cpp/src/barretenberg/crypto/schnorr/proof_of_possession.hpp b/cpp/src/barretenberg/crypto/schnorr/proof_of_possession.hpp index 69efb5cb8c..087e3ff37a 100644 --- a/cpp/src/barretenberg/crypto/schnorr/proof_of_possession.hpp +++ b/cpp/src/barretenberg/crypto/schnorr/proof_of_possession.hpp @@ -2,7 +2,7 @@ #include -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "schnorr.hpp" namespace crypto::schnorr { diff --git a/cpp/src/barretenberg/crypto/schnorr/schnorr.hpp b/cpp/src/barretenberg/crypto/schnorr/schnorr.hpp index ccd88dceb7..c91fe20d77 100644 --- a/cpp/src/barretenberg/crypto/schnorr/schnorr.hpp +++ b/cpp/src/barretenberg/crypto/schnorr/schnorr.hpp @@ -8,7 +8,7 @@ #include "barretenberg/crypto/hashers/hashers.hpp" -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/common/streams.hpp" namespace crypto { diff --git a/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp b/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp index 48debb4db9..5bee0a65cb 100644 --- a/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp +++ b/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp @@ -3,8 +3,13 @@ #include #include #include "barretenberg/common/streams.hpp" +#include "barretenberg/serialize/test_helper.hpp" #include "ecdsa_secp256k1.hpp" - +TEST(acir_format, msgpack_logic_constraint) +{ + auto [actual, expected] = msgpack_roundtrip(acir_format::LogicConstraint {}); + EXPECT_EQ(actual, expected); +} TEST(acir_format, test_logic_gate_from_noir_circuit) { /** diff --git a/cpp/src/barretenberg/dsl/acir_format/range_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/range_constraint.hpp index 7f5a512ce6..af769b8e06 100644 --- a/cpp/src/barretenberg/dsl/acir_format/range_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/range_constraint.hpp @@ -1,6 +1,6 @@ #pragma once #include -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" namespace acir_format { diff --git a/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp b/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp index 32885d81f1..83e29fd692 100644 --- a/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp +++ b/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp @@ -60,4 +60,12 @@ class Bn254FqParams { typedef field fq; -} // namespace barretenberg \ No newline at end of file +} // namespace barretenberg + +namespace msgpack { +// define this as a named alias in msgpack schema generation +inline void schema_pack(auto& packer, barretenberg::fq const&) +{ + packer.pack_alias("Fq", "bin32"); +} +} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/ecc/curves/bn254/fq.test.cpp b/cpp/src/barretenberg/ecc/curves/bn254/fq.test.cpp index 79e7bdea70..ce5e663236 100644 --- a/cpp/src/barretenberg/ecc/curves/bn254/fq.test.cpp +++ b/cpp/src/barretenberg/ecc/curves/bn254/fq.test.cpp @@ -1,9 +1,16 @@ #include "fq.hpp" #include "pseudorandom.hpp" +#include "barretenberg/serialize/test_helper.hpp" #include using namespace barretenberg; +TEST(fq, msgpack) +{ + auto [actual, expected] = msgpack_roundtrip(barretenberg::fq{1ull, 2ull, 3ull, 4ull}); + EXPECT_EQ(actual, expected); +} + TEST(fq, eq) { constexpr fq a{ 0x01, 0x02, 0x03, 0x04 }; diff --git a/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp b/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp index 0c2f11910b..dcbfde4423 100644 --- a/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp +++ b/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp @@ -65,4 +65,12 @@ class Bn254FrParams { typedef field fr; -} // namespace barretenberg \ No newline at end of file +} // namespace barretenberg + +namespace msgpack { +// define this as a named alias in msgpack schema generation +inline void schema_pack(auto& packer, barretenberg::fr const&) +{ + packer.pack_alias("Fr", "bin32"); +} +} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/ecc/curves/bn254/fr.test.cpp b/cpp/src/barretenberg/ecc/curves/bn254/fr.test.cpp index 683b491718..28cd331e1b 100644 --- a/cpp/src/barretenberg/ecc/curves/bn254/fr.test.cpp +++ b/cpp/src/barretenberg/ecc/curves/bn254/fr.test.cpp @@ -1,8 +1,15 @@ #include "fr.hpp" +#include "barretenberg/serialize/test_helper.hpp" #include using namespace barretenberg; +TEST(fr, msgpack) +{ + auto [actual, expected] = msgpack_roundtrip(barretenberg::fr{1ull, 2ull, 3ull, 4ull}); + EXPECT_EQ(actual, expected); +} + TEST(fr, eq) { fr a{ 0x01, 0x02, 0x03, 0x04 }; diff --git a/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp b/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp index dcabc0e3a9..17213910e4 100644 --- a/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp +++ b/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp @@ -17,4 +17,12 @@ struct Bn254G1Params { }; typedef group g1; -} // namespace barretenberg \ No newline at end of file + +} // namespace barretenberg + +namespace msgpack { +// specialize the name in msgpack schema generation +inline std::string schema_name(barretenberg::g1::affine_element const &) { + return "G1AffineElement"; +} +} \ No newline at end of file diff --git a/cpp/src/barretenberg/ecc/fields/.test.cpp b/cpp/src/barretenberg/ecc/fields/.test.cpp deleted file mode 100644 index ff01cf4425..0000000000 --- a/cpp/src/barretenberg/ecc/fields/.test.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "field.hpp" -#include -#include "barretenberg/common/msgpack_impl.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" - -TEST(field_msgpack, basic) -{ - barretenberg::fr field {1ull, 2ull, 3ull, 4ull}; - barretenberg::fr result {0, 0, 0, 0}; - msgpack::sbuffer buffer; - msgpack::pack(buffer, field); - msgpack::unpack(buffer.data(), buffer.size()).get().convert(result); - EXPECT_EQ(field, result); -} diff --git a/cpp/src/barretenberg/ecc/fields/field.hpp b/cpp/src/barretenberg/ecc/fields/field.hpp index 05d201c479..3ebaecd5db 100644 --- a/cpp/src/barretenberg/ecc/fields/field.hpp +++ b/cpp/src/barretenberg/ecc/fields/field.hpp @@ -2,7 +2,7 @@ #include #include "barretenberg/common/assert.hpp" #include "barretenberg/common/inline.hpp" -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include #include #include "barretenberg/numeric/random/engine.hpp" diff --git a/cpp/src/barretenberg/ecc/groups/affine_element.hpp b/cpp/src/barretenberg/ecc/groups/affine_element.hpp index c747d7cb36..3c95613ef3 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element.hpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element.hpp @@ -3,7 +3,7 @@ #include #include #include "barretenberg/ecc/curves/bn254/fq2.hpp" -#include "barretenberg/common/msgpack.hpp" +#include "barretenberg/serialize/msgpack.hpp" namespace barretenberg { namespace group_elements { @@ -179,7 +179,6 @@ template class alignas(64) affine_el Fq y; // for serialization: update up with new fields MSGPACK(x, y); - }; template void read(B& it, affine_element& value) diff --git a/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp b/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp index b7dcf57d46..7da7c62517 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp @@ -4,7 +4,8 @@ #include "barretenberg/ecc/curves/secp256r1/secp256r1.hpp" #include "barretenberg/common/test.hpp" #include -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/serialize/test_helper.hpp" namespace test_affine_element { template class test_affine_element : public testing::Test { @@ -122,4 +123,10 @@ TEST(affine_element, infinity_ordering_regression) P.self_set_infinity(); EXPECT_NE(P < Q, Q < P); } + +TEST(affine_element, msgpack) +{ + auto [actual, expected] = msgpack_roundtrip(secp256k1::g1::affine_element {}); + EXPECT_EQ(actual, expected); +} } // namespace test_affine_element diff --git a/cpp/src/barretenberg/ecc/serialize.test.cpp b/cpp/src/barretenberg/ecc/serialize.test.cpp new file mode 100644 index 0000000000..6a685d3305 --- /dev/null +++ b/cpp/src/barretenberg/ecc/serialize.test.cpp @@ -0,0 +1,9 @@ +#include +#include "barretenberg/serialize/test_helper.hpp" +#include "barretenberg/ecc/fields/field.hpp" + +TEST(msgpack_tests, msgpack_field) +{ + auto [actual, expected] = msgpack_roundtrip(barretenberg::fr{1ull, 2ull, 3ull, 4ull}); + EXPECT_EQ(actual, expected); +} \ No newline at end of file diff --git a/cpp/src/barretenberg/honk/sumcheck/polynomials/univariate.hpp b/cpp/src/barretenberg/honk/sumcheck/polynomials/univariate.hpp index a3cc9cf82f..a502aeefb2 100644 --- a/cpp/src/barretenberg/honk/sumcheck/polynomials/univariate.hpp +++ b/cpp/src/barretenberg/honk/sumcheck/polynomials/univariate.hpp @@ -3,7 +3,7 @@ #include #include #include -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/common/assert.hpp" namespace proof_system::honk::sumcheck { diff --git a/cpp/src/barretenberg/honk/sumcheck/sumcheck.hpp b/cpp/src/barretenberg/honk/sumcheck/sumcheck.hpp index 8968ee1260..40c905f29a 100644 --- a/cpp/src/barretenberg/honk/sumcheck/sumcheck.hpp +++ b/cpp/src/barretenberg/honk/sumcheck/sumcheck.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include #include "barretenberg/honk/sumcheck/relations/relation.hpp" #include "barretenberg/honk/transcript/transcript.hpp" diff --git a/cpp/src/barretenberg/honk/transcript/transcript.hpp b/cpp/src/barretenberg/honk/transcript/transcript.hpp index fec507a642..fe7b078200 100644 --- a/cpp/src/barretenberg/honk/transcript/transcript.hpp +++ b/cpp/src/barretenberg/honk/transcript/transcript.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/crypto/blake3s/blake3s.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/bridge_call_data.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/bridge_call_data.hpp index 22d79996ad..5f93f98334 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/bridge_call_data.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/bridge_call_data.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "../constants.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note.hpp index a89e12bcfb..67f573ab09 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "create_partial_commitment.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note_tx_data.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note_tx_data.hpp index 83c2f6375c..5e545f10e6 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note_tx_data.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note_tx_data.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "../bridge_call_data.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/complete_partial_commitment.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/complete_partial_commitment.hpp index 27407bf567..dd37fd1b31 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/complete_partial_commitment.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/complete_partial_commitment.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "../../constants.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/compute_nullifier.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/compute_nullifier.hpp index 513b8bfe2e..acd08bc934 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/compute_nullifier.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/compute_nullifier.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "../../constants.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/create_partial_commitment.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/create_partial_commitment.hpp index 5105074c02..a5c941a7ea 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/create_partial_commitment.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/create_partial_commitment.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "../../constants.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/value_note.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/value_note.hpp index 421106004d..df4367ac49 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/value_note.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/value_note.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "create_partial_commitment.hpp" diff --git a/cpp/src/barretenberg/numeric/uint128/uint128.hpp b/cpp/src/barretenberg/numeric/uint128/uint128.hpp index 884c73e9c6..27ba089613 100644 --- a/cpp/src/barretenberg/numeric/uint128/uint128.hpp +++ b/cpp/src/barretenberg/numeric/uint128/uint128.hpp @@ -4,7 +4,7 @@ #ifdef __i386__ #include -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" namespace numeric { diff --git a/cpp/src/barretenberg/numeric/uint256/uint256.hpp b/cpp/src/barretenberg/numeric/uint256/uint256.hpp index 52c6411e1c..1c0d8f1cde 100644 --- a/cpp/src/barretenberg/numeric/uint256/uint256.hpp +++ b/cpp/src/barretenberg/numeric/uint256/uint256.hpp @@ -15,7 +15,7 @@ #include #include #include -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" #include "../uint128/uint128.hpp" namespace numeric { diff --git a/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp b/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp index ef1d1c1392..2b05d87bd2 100644 --- a/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp +++ b/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp @@ -1,13 +1,13 @@ #include "prover.hpp" #include "barretenberg/env/data_store.hpp" #include "barretenberg/env/crs.hpp" +#include "barretenberg/serialize/test_helper.hpp" +#include "barretenberg/serialize/raw_pointer.hpp" -#define WASM_EXPORT __attribute__((visibility("default"))) +#define WASM_EXPORT extern "C" __attribute__((visibility("default"))) using namespace barretenberg; -extern "C" { - /** * Called by `barretenberg_wasm.test.ts` to test the asyncify intrumentation and logic that * allows for WASM code to make calls to async code in JS. @@ -50,6 +50,14 @@ WASM_EXPORT void* test_env_load_prover_crs(size_t num_points) using WasmProver = plonk::UltraProver; +typedef RawPointer WasmProverPtr; + +// TODO(AD): Currently just a motivating example, TODO bind rest of library +CBIND(prover_process_queue2, [](WasmProverPtr prover) { + prover->queue.process_queue(); + return 0; +}); + WASM_EXPORT void prover_process_queue(WasmProver* prover) { prover->queue.process_queue(); @@ -168,4 +176,3 @@ WASM_EXPORT void delete_evaluation_domain(void* domain) { delete reinterpret_cast(domain); } -} diff --git a/cpp/src/barretenberg/plonk/proof_system/proving_key/serialize.hpp b/cpp/src/barretenberg/plonk/proof_system/proving_key/serialize.hpp index 6bc7e79ed5..67d7c89066 100644 --- a/cpp/src/barretenberg/plonk/proof_system/proving_key/serialize.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/proving_key/serialize.hpp @@ -2,7 +2,7 @@ #include "proving_key.hpp" #include "barretenberg/polynomials/serialize.hpp" #include "barretenberg/common/throw_or_abort.hpp" -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" namespace proof_system::plonk { diff --git a/cpp/src/barretenberg/plonk/proof_system/types/polynomial_manifest.hpp b/cpp/src/barretenberg/plonk/proof_system/types/polynomial_manifest.hpp index 8a56d34168..c16aeaafdf 100644 --- a/cpp/src/barretenberg/plonk/proof_system/types/polynomial_manifest.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/types/polynomial_manifest.hpp @@ -255,4 +255,4 @@ class PrecomputedPolyList { std::string operator[](size_t index) const { return precomputed_poly_ids[index]; } }; -} // namespace proof_system::plonk +} // namespace proof_system::plonk \ No newline at end of file diff --git a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp index 818dc643e4..3c7606d797 100644 --- a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp @@ -3,8 +3,8 @@ #include #include #include -#include "barretenberg/common/serialize.hpp" -#include "barretenberg/common/msgpack.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/serialize/msgpack.hpp" namespace proof_system::plonk { @@ -49,3 +49,11 @@ inline std::ostream& operator<<(std::ostream& os, proof const& data) } } // namespace proof_system::plonk + +namespace msgpack { +// help our msgpack schema compiler with this typedef +inline void schema_pack(auto& packer, proof_system::plonk::proof const&) +{ + packer.pack_alias("Proof", "bin32"); +} +} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp index 8c2cf050dc..a04e5af793 100644 --- a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp @@ -7,7 +7,7 @@ #include "barretenberg/polynomials/evaluation_domain.hpp" #include "barretenberg/crypto/sha256/sha256.hpp" #include "barretenberg/plonk/proof_system/types/polynomial_manifest.hpp" -#include "barretenberg/common/msgpack.hpp" +#include "barretenberg/serialize/msgpack.hpp" namespace proof_system::plonk { @@ -115,6 +115,12 @@ struct verification_key { } }; +namespace msgpack { +inline void msgpack_schema(auto &packer, proof_system::plonk::verification_key const &) { + packer.pack_schema(proof_system::plonk::verification_key_data{}); +} +} // namespace msgpack + template inline void read(B& buf, verification_key& key) { auto env_crs = std::make_unique(); diff --git a/cpp/src/barretenberg/proof_system/arithmetization/gate_data.hpp b/cpp/src/barretenberg/proof_system/arithmetization/gate_data.hpp index 5175867bfd..70740cbe1b 100644 --- a/cpp/src/barretenberg/proof_system/arithmetization/gate_data.hpp +++ b/cpp/src/barretenberg/proof_system/arithmetization/gate_data.hpp @@ -1,7 +1,7 @@ #pragma once #include #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/common/serialize.hpp" +#include "barretenberg/serialize/legacy_serialize.hpp" namespace proof_system { struct add_triple { diff --git a/cpp/src/barretenberg/serialize/CMakeLists.txt b/cpp/src/barretenberg/serialize/CMakeLists.txt new file mode 100644 index 0000000000..f7c49f1111 --- /dev/null +++ b/cpp/src/barretenberg/serialize/CMakeLists.txt @@ -0,0 +1,7 @@ +# For running tests only, not to be depended on +# The non-test portion of barretenberg (the part you actually use) is header only +barretenberg_module( + serialize-tests + proof_system + transcript +) \ No newline at end of file diff --git a/cpp/src/barretenberg/serialize/cbind.hpp b/cpp/src/barretenberg/serialize/cbind.hpp new file mode 100644 index 0000000000..da25c49d76 --- /dev/null +++ b/cpp/src/barretenberg/serialize/cbind.hpp @@ -0,0 +1,139 @@ +#pragma once +// Meant to be the main header included by translation units that use msgpack. +// Note: heavy header due to serialization logic, don't include if msgpack.hpp will do +// CBinding helpers that take a function or a lambda and +// - bind the input as a coded msgpack array of all the arguments (using template metamagic) +// - bind the return value to an out buffer, where the caller must free the memory + +#include "msgpack_impl/check_memory_span.hpp" +#include "msgpack_impl/concepts.hpp" +#include "msgpack_impl/msgpack_impl.hpp" +#include "msgpack_impl/name_value_pair_macro.hpp" +#include "msgpack_impl/schema_impl.hpp" +#include "msgpack_impl/schema_name.hpp" +#include "msgpack_impl/struct_map_impl.hpp" +#include "msgpack_impl/variant_impl.hpp" + +#include +#include + +namespace msgpack { +inline std::pair encode_buffer(auto&& obj) +{ + // Create a buffer to store the encoded data + msgpack::sbuffer buffer; + msgpack::pack(buffer, obj); + + uint8_t* output = (uint8_t*)aligned_alloc(64, buffer.size()); + memcpy(output, buffer.data(), buffer.size()); + // Convert the buffer data to a string and return it + return { output, buffer.size() }; +} + +template inline void decode(T* value, const uint8_t* encoded_data, size_t encoded_data_size) +{ + // Create a MsgPack unpacker + msgpack::unpack((const char*)encoded_data, encoded_data_size).get().convert(*value); +} + +std::string string_encode(const auto& obj) +{ + std::stringstream output; + msgpack::pack(output, obj); + std::string output_str = output.str(); + msgpack::object_handle oh = msgpack::unpack(output_str.data(), output_str.size()); + std::stringstream pretty_output; + pretty_output << oh.get() << std::endl; + return pretty_output.str(); +} +void print(const auto& obj) +{ + std::stringstream output; + msgpack::pack(output, obj); + std::string output_str = output.str(); + msgpack::object_handle oh = msgpack::unpack(output_str.data(), output_str.size()); + std::stringstream pretty_output; + pretty_output << oh.get() << std::endl; + logstr(pretty_output.str().c_str()); +} + +template struct func_traits; + +template struct func_traits { + typedef std::tuple Args; + Args args; + R ret; + MSGPACK(args, ret); +}; +template struct func_traits { + typedef std::tuple Args; + Args args; + R ret; + MSGPACK(args, ret); +}; + +template struct func_traits { + typedef std::tuple Args; + Args args; + R ret; + MSGPACK(args, ret); +}; + +// Template metamagic for figuring out the parameters and return type of a function +template +concept Callable = + requires() { typename std::enable_if_t, void>; }; + +template constexpr auto get_func_traits() +{ + return func_traits(); +} + +template constexpr auto get_func_traits() +{ + return func_traits(); +} + +template constexpr auto param_tuple() +{ + return typename decltype(get_func_traits())::Args{}; +} + +inline void cbind_impl( + auto func, const uint8_t* input_in, size_t input_len_in, uint8_t** output_out, size_t* output_len_out) +{ + auto params = param_tuple(); + msgpack::decode(¶ms, input_in, input_len_in); + auto [output, output_len] = msgpack::encode_buffer(std::apply(func, params)); + *output_out = output; + *output_len_out = output_len; +} + +// returns a C-style string json of the schema +inline void cbind_schema_impl(auto func, uint8_t** output_out, size_t* output_len_out) +{ + (void)func; // unused except for type + // Object representation of the cbind + auto cbind_obj = get_func_traits(); + std::string schema = msgpack::schema_to_string(cbind_obj); + *output_out = (uint8_t*)aligned_alloc(64, schema.size() + 1); + memcpy(*output_out, schema.c_str(), schema.size() + 1); + *output_len_out = schema.size(); +} +} // namespace msgpack + + +// The CBIND macro is a convenient utility that abstracts away several steps in binding C functions with msgpack +// serialization. It creates two separate functions: +// 1. cname function: This decodes the input arguments from msgpack format, calls the target function, +// and then encodes the return value back into msgpack format. +// 2. cname##__schema function: This creates a JSON schema of the function's input arguments and return type. +#define CBIND(cname, func) \ + WASM_EXPORT void cname(const uint8_t* input_in, size_t input_len_in, uint8_t** output_out, size_t* output_len_out) \ + { \ + msgpack::cbind_impl(func, input_in, input_len_in, output_out, output_len_out); \ + } \ + WASM_EXPORT void cname##__schema(uint8_t** output_out, size_t* output_len_out) \ + { \ + msgpack::cbind_schema_impl(func, output_out, output_len_out); \ + } \ No newline at end of file diff --git a/cpp/src/barretenberg/serialize/cbind_fwd.h b/cpp/src/barretenberg/serialize/cbind_fwd.h new file mode 100644 index 0000000000..514271f437 --- /dev/null +++ b/cpp/src/barretenberg/serialize/cbind_fwd.h @@ -0,0 +1,10 @@ +#pragma once +#include +#include +// CBIND forward declarations for msgback default bind format (encode as tuple of args and return value as msgpack string) +#define WASM_EXPORT extern "C" __attribute__((visibility("default"))) + +#define CBIND_DECL(cname) \ + WASM_EXPORT void cname(const uint8_t* input_in, size_t input_len_in, uint8_t** output_out, size_t* output_len_out);\ + WASM_EXPORT void cname##__schema(uint8_t** output_out, size_t* output_len_out); \ + std::pair cname##__test() \ No newline at end of file diff --git a/cpp/src/barretenberg/common/serialize.hpp b/cpp/src/barretenberg/serialize/legacy_serialize.hpp similarity index 98% rename from cpp/src/barretenberg/common/serialize.hpp rename to cpp/src/barretenberg/serialize/legacy_serialize.hpp index f3ef628fea..126097470c 100644 --- a/cpp/src/barretenberg/common/serialize.hpp +++ b/cpp/src/barretenberg/serialize/legacy_serialize.hpp @@ -1,5 +1,6 @@ /** - * This is the core serialization library. + * This is a non-msgpack flat buffer serialization library. + * It is currently used alongside msgpack, with hope to eventually move to msgpack. * It enables the reading and writing of big-endian formatted integers and various standard library types * to and from the following supported types: * - uint8_t* diff --git a/cpp/src/barretenberg/common/msgpack.hpp b/cpp/src/barretenberg/serialize/msgpack.hpp similarity index 93% rename from cpp/src/barretenberg/common/msgpack.hpp rename to cpp/src/barretenberg/serialize/msgpack.hpp index b93de50281..8d0d917cfb 100644 --- a/cpp/src/barretenberg/common/msgpack.hpp +++ b/cpp/src/barretenberg/serialize/msgpack.hpp @@ -25,7 +25,7 @@ void msgpack(auto ar) { recursive_proof_public_input_indices)); } or -MSGPACK(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, +MSGPACK_FIELDS(composer_type, circuit_size, num_public_inputs, commitments, contains_recursive_proof, recursive_proof_public_input_indices); ``` @@ -69,7 +69,7 @@ These methods allow you to implement custom logic for the serialization and dese ## Packing/Unpacking -Only when actually using msgpack to write or read data, include "barretenberg/common/msgpack_impl.hpp". +Only when actually using msgpack to write or read data, include "barretenberg/serialize/cbind.hpp". You can then use msgpack library features to serialize and deserialize C++ objects. e.g. packing @@ -91,8 +91,8 @@ e.g. unpacking ``` */ #include -#include "msgpack/name_value_pair_macro.hpp" -#include "msgpack/concepts.hpp" +#include "msgpack_impl/name_value_pair_macro.hpp" +#include "msgpack_impl/concepts.hpp" // Helper for above documented syntax #define MSGPACK(...) \ diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/check_memory_span.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/check_memory_span.hpp new file mode 100644 index 0000000000..e107b577e4 --- /dev/null +++ b/cpp/src/barretenberg/serialize/msgpack_impl/check_memory_span.hpp @@ -0,0 +1,76 @@ +#pragma once +// Note: heavy header due to serialization logic, don't include outside of tests +#include + +#include +#include +#include +#include +#include +#include "barretenberg/common/throw_or_abort.hpp" +#include "schema_name.hpp" + +namespace msgpack { +template uintptr_t __aligned_for(uintptr_t ptr) +{ + // Round to next alignment, (ptr % alignof(T)) == 0 after + return ptr + (alignof(T) - (ptr % alignof(T))) % alignof(T); +} +template std::string check_memory_span(T* obj, Args*... args) +{ + // We need to handle alignment. Thankfully, we have a tool here. + // Convert the variadic template arguments to a vector of pairs. + // Each pair contains a pointer (as uintptr_t) and its size. + std::vector> pointers{ { (uintptr_t)(args), sizeof(Args) }... }; + // Sort the vector based on the pointer values. + std::sort(pointers.begin(), pointers.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); + + for (size_t i = 1; i < pointers.size(); ++i) { + // Check if any of the Args* pointers overlap. + auto last_end = pointers[i - 1].first + pointers[i - 1].second; + if (last_end > pointers[i].first) { + return "Overlap in " + msgpack::schema_name(*obj) + " MSGPACK() params detected!"; + } + // Check if gap is too large. + // Give some fuzzy room in case of 64 byte alignment restrictions. + if (__aligned_for(last_end) < pointers[i].first) { + return "Gap in " + msgpack::schema_name(*obj) + " MSGPACK() params detected before member #" + + std::to_string(i) + " !"; + } + } + + // Check if all Args* pointers exist in T* memory. + uintptr_t t_start = reinterpret_cast(obj); + uintptr_t t_end = t_start + sizeof(T); + if (pointers.front().first < t_start || pointers.back().first + pointers.back().second > t_end) { + return "Some " + msgpack::schema_name(*obj) + " MSGPACK() params don't exist in object!"; + } + + // Check if all of T* memory is used by the Args* pointers. + size_t start = (size_t)obj; + size_t end = (size_t)obj; + for (auto [ptr, size] : pointers) { + end = std::max(end, ptr + size); + } + size_t total_size = end - start; + if (__aligned_for(total_size) < sizeof(T)) { + return "Incomplete " + msgpack::schema_name(*obj) + " MSGPACK() params! Not all of object specified."; + } + return {}; +} + +template std::string check_msgpack_method(T& object) +{ + std::string result; + auto checker = [&](auto&... values) { result = check_memory_span(&object, &values...); }; + object.msgpack([&](auto&... keys_and_values) { std::apply(checker, drop_keys(std::tie(keys_and_values...))); }); + return result; +} +void check_msgpack_usage(auto object) +{ + std::string result = check_msgpack_method(object); + if (!result.empty()) { + throw_or_abort(result); + } +} +} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/common/msgpack/concepts.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp similarity index 100% rename from cpp/src/barretenberg/common/msgpack/concepts.hpp rename to cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp diff --git a/cpp/src/barretenberg/common/msgpack_impl.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/msgpack_impl.hpp similarity index 54% rename from cpp/src/barretenberg/common/msgpack_impl.hpp rename to cpp/src/barretenberg/serialize/msgpack_impl/msgpack_impl.hpp index b2272ec89d..1db14a898a 100644 --- a/cpp/src/barretenberg/common/msgpack_impl.hpp +++ b/cpp/src/barretenberg/serialize/msgpack_impl/msgpack_impl.hpp @@ -1,4 +1,4 @@ #pragma once // Note: Meant to only be included in compilation units that need msgpack -#include "msgpack/struct_map_impl.hpp" -#include "msgpack/variant_impl.hpp" \ No newline at end of file +#include "struct_map_impl.hpp" +#include "variant_impl.hpp" \ No newline at end of file diff --git a/cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/name_value_pair_macro.hpp similarity index 100% rename from cpp/src/barretenberg/common/msgpack/name_value_pair_macro.hpp rename to cpp/src/barretenberg/serialize/msgpack_impl/name_value_pair_macro.hpp diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp new file mode 100644 index 0000000000..08e74d0079 --- /dev/null +++ b/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp @@ -0,0 +1,176 @@ +#pragma once + +#include +#include +#include +#include +#include "schema_name.hpp" + +namespace msgpack { + +/** + * Define a serialization schema based on compile-time information about a type being serialized. + * This is then consumed by typescript to make bindings. + */ +struct SchemaPacker : packer { + SchemaPacker(msgpack::sbuffer& stream) + : packer(stream) + {} + // For tracking emitted types + std::set emitted_types; + // Returns if already was emitted + bool set_emitted(const std::string& type) + { + if (emitted_types.find(type) == emitted_types.end()) { + emitted_types.insert(type); + return false; + } + return true; + } + + /** + * Pack a type indicating it is an alias of a certain msgpack type + * Packs in the form ["alias", [schema_name, msgpack_name]] + * @param schema_name The CPP type. + * @param msgpack_name The msgpack type. + */ + void pack_alias(const std::string& schema_name, const std::string& msgpack_name) + { + pack_array(2); + pack("alias"); + pack_array(2); + pack(schema_name); + pack(msgpack_name); + } + + /** + * Pack the schema of a given object. + * @tparam T the object's type. + * @param obj the object. + */ + template + void pack_schema(const T& obj) + { + schema_pack(*this, obj); + } +}; + +// Helper for packing (key, value, key, value, ...) arguments +inline void _schema_pack_map_content(SchemaPacker&) +{ + // base case +} +// Helper for packing (key, value, key, value, ...) arguments +template +inline void _schema_pack_map_content(SchemaPacker& packer, std::string key, Value value, Rest... rest) +{ + packer.pack(key); + schema_pack(packer, value); + _schema_pack_map_content(packer, rest...); +} + +/** + * Schema pack base case for types with no special msgpack method. + * @tparam T the type. + * @param packer the schema packer. + */ +template + requires(!HasMsgPack && !HasMsgPackPack) +inline void schema_pack(SchemaPacker& packer, T const&) +{ + packer.pack(schema_name(T{})); +} + +/** + * @brief Encode a type that defines msgpack based on its key value pairs. + * + * @tparam T the msgpack()'able type + * @param packer Our special packer. + * @param object The object in question. + */ +template inline void schema_pack(SchemaPacker& packer, T const& object) +{ + std::string type = schema_name(object); + if (packer.set_emitted(type)) { + packer.pack(type); + return; // already emitted + } + msgpack::check_msgpack_usage(object); + // Encode as map + const_cast(object).msgpack([&](auto&... args) { + size_t kv_size = sizeof...(args); + packer.pack_map(uint32_t(1 + kv_size / 2)); + packer.pack("__typename"); + packer.pack(type); + _schema_pack_map_content(packer, args...); + }); +} + +// Alias verification_key as verification_key_data +//inline void schema_pack(SchemaPacker& packer, proof_system::plonk::verification_key const&) +//{ +// schema_pack(packer, proof_system::plonk::verification_key_data{}); +//} + +// Recurse over any templated containers +// Outputs e.g. ['vector', ['sub-type']] +template inline void _schema_pack(SchemaPacker& packer, const std::string& schema_name) +{ + packer.pack_array(2); + packer.pack(schema_name); + packer.pack_array(sizeof...(Args)); + (schema_pack(packer, Args{}), ...); /* pack schemas of all template Args */ +} +template inline void schema_pack(SchemaPacker& packer, std::tuple const&) +{ + _schema_pack(packer, "tuple"); +} +template inline void schema_pack(SchemaPacker& packer, std::map const&) +{ + _schema_pack(packer, "map"); +} +template inline void schema_pack(SchemaPacker& packer, std::optional const&) +{ + _schema_pack(packer, "optional"); +} +template inline void schema_pack(SchemaPacker& packer, std::vector const&) +{ + _schema_pack(packer, "vector"); +} +template inline void schema_pack(SchemaPacker& packer, std::variant const&) +{ + _schema_pack(packer, "variant"); +} +template inline void schema_pack(SchemaPacker& packer, std::shared_ptr const&) +{ + _schema_pack(packer, "shared_ptr"); +} + +// Outputs e.g. ['array', ['array-type', 'N']] +template inline void schema_pack(SchemaPacker& packer, std::array const&) +{ + packer.pack_array(2); + packer.pack("array"); + packer.pack_array(2); /* param list format for consistency*/ + schema_pack(packer, T{}); + packer.pack(N); +} + +/** + * @brief Print's an object's derived msgpack schema as a string. + * + * @param obj The object to print schema of. + * @return std::string The schema as a string. + */ +std::string schema_to_string(auto obj) +{ + msgpack::sbuffer output; + SchemaPacker printer{ output }; + schema_pack(printer, obj); + msgpack::object_handle oh = msgpack::unpack(output.data(), output.size()); + std::stringstream pretty_output; + pretty_output << oh.get() << std::endl; + return pretty_output.str(); +} + +} // namespace msgpack diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/schema_name.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/schema_name.hpp new file mode 100644 index 0000000000..7c1f46961e --- /dev/null +++ b/cpp/src/barretenberg/serialize/msgpack_impl/schema_name.hpp @@ -0,0 +1,33 @@ +#pragma once +#include "barretenberg/ecc/curves/bn254/g1.hpp" +#include +#include + +namespace msgpack { +/** + * Converts C++ type information into a human-readable format. + * This function leverages __cxa_demangle to demangle the name generated by typeid. + * Special cases are handled for "basic_string" (returns "string") and "i" (returns "int"). + * Template specializations are truncated and only the base type name is returned + * @tparam T the type. + * @return the readable schema name. + */ +template std::string schema_name(T const&) +{ + std::string result = abi::__cxa_demangle(typeid(T).name(), NULL, NULL, NULL); + if (result.find("basic_string") != std::string::npos) { + return "string"; + } + if (result == "i") { + return "int"; + } + + if (result.find('<') != size_t(-1)) { + result = result.substr(0, result.find('<')); + } + if (result.rfind(':') != size_t(-1)) { + result = result.substr(result.rfind(':') + 1, result.size()); + } + return result; +} +} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp similarity index 100% rename from cpp/src/barretenberg/common/msgpack/struct_map_impl.hpp rename to cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp diff --git a/cpp/src/barretenberg/common/msgpack/variant_impl.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/variant_impl.hpp similarity index 100% rename from cpp/src/barretenberg/common/msgpack/variant_impl.hpp rename to cpp/src/barretenberg/serialize/msgpack_impl/variant_impl.hpp diff --git a/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp b/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp new file mode 100644 index 0000000000..4d8d5476d1 --- /dev/null +++ b/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp @@ -0,0 +1,86 @@ +#include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/serialize/cbind.hpp" + +#include + +// Sanity checking for msgpack +// TODO eventually move to barretenberg + +struct GoodExample { + barretenberg::fr a; + barretenberg::fr b; + MSGPACK(a, b); +} good_example; + +struct BadExampleOverlap { + barretenberg::fr a; + barretenberg::fr b; + MSGPACK(a, a); +} bad_example_overlap; + +struct BadExampleIncomplete { + barretenberg::fr a; + barretenberg::fr b; + MSGPACK(a); +} bad_example_incomplete; + +struct BadExampleCompileTimeError { + std::vector a; + barretenberg::fr b; + + MSGPACK(b); // Type mismatch, expect 'a', will catch at compile-time +} bad_example_compile_time_error; + +struct BadExampleOutOfObject { + barretenberg::fr a; + barretenberg::fr b; + void msgpack(auto ar) + { + BadExampleOutOfObject other_object; + ar("a", other_object.a, "b", other_object.b); + } +} bad_example_out_of_object; + +// TODO eventually move to barretenberg +TEST(msgpack_tests, msgpack_sanity_sanity) +{ + EXPECT_EQ(msgpack::check_msgpack_method(good_example), ""); + EXPECT_EQ(msgpack::check_msgpack_method(bad_example_overlap), + "Overlap in BadExampleOverlap MSGPACK() params detected!"); + EXPECT_EQ(msgpack::check_msgpack_method(bad_example_incomplete), + "Incomplete BadExampleIncomplete MSGPACK() params! Not all of object specified."); + + // If we actually try to msgpack BadExampleCompileTimeError we will statically error + // This is great, but we need to check the underlying facility *somehow* + auto checker = [&](auto&... values) { + std::string incomplete_msgpack_status = "error"; + if constexpr (msgpack::MsgpackConstructible) { + incomplete_msgpack_status = ""; + } + EXPECT_EQ(incomplete_msgpack_status, "error"); + }; + bad_example_compile_time_error.msgpack(checker); + + EXPECT_EQ(msgpack::check_msgpack_method(bad_example_out_of_object), + "Some BadExampleOutOfObject MSGPACK() params don't exist in object!"); +} + +struct ComplicatedSchema { + std::vector> array; + std::optional good_or_not; + barretenberg::fr bare; + std::variant huh; + MSGPACK(array, good_or_not, bare, huh); +} complicated_schema; + +TEST(msgpack_tests, msgpack_schema_sanity) +{ + EXPECT_EQ( + msgpack::schema_to_string(good_example), + "{\"__typename\":\"GoodExample\",\"a\":[\"alias\",[\"Fr\",\"bin32\"]],\"b\":[\"alias\",[\"Fr\",\"bin32\"]]}\n"); + EXPECT_EQ(msgpack::schema_to_string(complicated_schema), + "{\"__typename\":\"ComplicatedSchema\",\"array\":[\"vector\",[[\"array\",[[\"alias\",[\"Fr\",\"bin32\"]]," + "20]]]],\"good_or_not\":[\"optional\",[{\"__typename\":\"GoodExample\",\"a\":[\"alias\",[\"Fr\"," + "\"bin32\"]],\"b\":[\"alias\",[\"Fr\",\"bin32\"]]}]],\"bare\":[\"alias\",[\"Fr\",\"bin32\"]],\"huh\":[" + "\"variant\",[[\"alias\",[\"Fr\",\"bin32\"]],\"GoodExample\"]]}\n"); +} diff --git a/cpp/src/barretenberg/serialize/raw_pointer.hpp b/cpp/src/barretenberg/serialize/raw_pointer.hpp new file mode 100644 index 0000000000..5b61b9ad1c --- /dev/null +++ b/cpp/src/barretenberg/serialize/raw_pointer.hpp @@ -0,0 +1,25 @@ +#include +#include "msgpack_impl/schema_name.hpp" + +template +struct RawPointer { + T* prover {}; + void msgpack_pack(auto& packer) const { + packer.pack(reinterpret_cast(prover)); + } + void msgpack_unpack(auto object) { + prover = reinterpret_cast((uintptr_t)object); + } + T* operator->() { + return prover; + } +}; + +namespace msgpack { +// help our msgpack schema compiler with this struct +template +inline void schema_pack(auto& packer, RawPointer const&) +{ + packer.pack_alias(schema_name(T {}) + "Ptr", "int"); +} +} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/serialize/test_helper.hpp b/cpp/src/barretenberg/serialize/test_helper.hpp new file mode 100644 index 0000000000..eaed342245 --- /dev/null +++ b/cpp/src/barretenberg/serialize/test_helper.hpp @@ -0,0 +1,29 @@ +#pragma once +#include +#include "cbind.hpp" + +/*** + * Do a roundtrip test encode/decode of an object. + * @tparam T The object type. + * @param object The object. Can be a default-initialized object. + */ +template +std::pair msgpack_roundtrip(const T &object) { + T result; + msgpack::sbuffer buffer; + msgpack::pack(buffer, object); + msgpack::unpack(buffer.data(), buffer.size()).get().convert(result); + return {object, result}; +} + +inline std::pair cbind_test_impl(auto cbind_func, auto... test_args) { + auto expected_ret = func(test_args...); + auto [input, input_len] = msgpack::encode_buffer(std::make_tuple(test_args...)); + uint8_t *output; + size_t output_len; + cbind_func(input, input_len, &output, &output_len); + decltype(expected_ret) actual_ret; + msgpack::decode(&actual_ret, output, output_len); + aligned_free(output); + return {msgpack::string_encode(actual_ret), msgpack::string_encode(expected_ret)}; +} diff --git a/cpp/src/barretenberg/stdlib/primitives/address/address.hpp b/cpp/src/barretenberg/stdlib/primitives/address/address.hpp index c4cfd3a83e..9eccf401e7 100644 --- a/cpp/src/barretenberg/stdlib/primitives/address/address.hpp +++ b/cpp/src/barretenberg/stdlib/primitives/address/address.hpp @@ -143,4 +143,12 @@ template class address_t { }; } // namespace stdlib -} // namespace proof_system::plonk \ No newline at end of file +} // namespace proof_system::plonk + +namespace msgpack { +// help our msgpack schema compiler with this buffer alias (as far as wire representation is concerned) class +inline void schema_pack(auto& packer, proof_system::plonk::stdlib::address const&) +{ + packer.pack_alias("Address", "bin32"); +} +} // namespace msgpack \ No newline at end of file From b32bd48acaf183dbb3a2654cd3f768031ce00738 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 09:25:31 -0400 Subject: [PATCH 65/92] Fix circuits build --- cpp/src/barretenberg/serialize/cbind_fwd.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/src/barretenberg/serialize/cbind_fwd.h b/cpp/src/barretenberg/serialize/cbind_fwd.h index 514271f437..fef437e4dc 100644 --- a/cpp/src/barretenberg/serialize/cbind_fwd.h +++ b/cpp/src/barretenberg/serialize/cbind_fwd.h @@ -6,5 +6,4 @@ #define CBIND_DECL(cname) \ WASM_EXPORT void cname(const uint8_t* input_in, size_t input_len_in, uint8_t** output_out, size_t* output_len_out);\ - WASM_EXPORT void cname##__schema(uint8_t** output_out, size_t* output_len_out); \ - std::pair cname##__test() \ No newline at end of file + WASM_EXPORT void cname##__schema(uint8_t** output_out, size_t* output_len_out); From 4a4e89937ca3b5500ba3b7dbccaba5f140436139 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 09:26:35 -0400 Subject: [PATCH 66/92] Try to make circuits test work again --- .../{serialize/legacy_serialize.hpp => common/serialize.hpp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cpp/src/barretenberg/{serialize/legacy_serialize.hpp => common/serialize.hpp} (100%) diff --git a/cpp/src/barretenberg/serialize/legacy_serialize.hpp b/cpp/src/barretenberg/common/serialize.hpp similarity index 100% rename from cpp/src/barretenberg/serialize/legacy_serialize.hpp rename to cpp/src/barretenberg/common/serialize.hpp From 5818f3a525f50c34dfb1aedd70c16e6fb3793f6c Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 09:38:32 -0400 Subject: [PATCH 67/92] Try to make circuits test work again --- cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp | 4 ++-- cpp/src/barretenberg/crypto/ecdsa/ecdsa_impl.hpp | 2 +- cpp/src/barretenberg/crypto/hmac/hmac.hpp | 2 +- .../barretenberg/crypto/pedersen_commitment/c_bind.cpp | 3 +-- .../barretenberg/crypto/pedersen_commitment/c_bind.hpp | 2 +- cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp | 2 +- cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp | 2 +- .../crypto/schnorr/proof_of_possession.hpp | 2 +- cpp/src/barretenberg/crypto/schnorr/schnorr.hpp | 2 +- .../barretenberg/dsl/acir_format/range_constraint.hpp | 2 +- cpp/src/barretenberg/ecc/fields/field.hpp | 2 +- .../barretenberg/ecc/groups/affine_element.test.cpp | 4 ++-- .../honk/sumcheck/polynomials/univariate.hpp | 2 +- cpp/src/barretenberg/honk/sumcheck/sumcheck.hpp | 2 +- cpp/src/barretenberg/honk/transcript/transcript.hpp | 2 +- .../proofs/notes/native/bridge_call_data.hpp | 2 +- .../proofs/notes/native/claim/claim_note.hpp | 2 +- .../proofs/notes/native/claim/claim_note_tx_data.hpp | 2 +- .../notes/native/claim/complete_partial_commitment.hpp | 2 +- .../proofs/notes/native/claim/compute_nullifier.hpp | 2 +- .../notes/native/value/create_partial_commitment.hpp | 2 +- .../proofs/notes/native/value/value_note.hpp | 2 +- cpp/src/barretenberg/numeric/uint128/uint128.hpp | 2 +- cpp/src/barretenberg/numeric/uint256/uint256.hpp | 2 +- .../plonk/proof_system/proving_key/serialize.hpp | 2 +- .../barretenberg/plonk/proof_system/types/proof.hpp | 10 +++------- .../proof_system/arithmetization/gate_data.hpp | 2 +- 27 files changed, 31 insertions(+), 36 deletions(-) diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp index fa38d0a738..fcab380014 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp @@ -1,7 +1,7 @@ #include "ecdsa.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "barretenberg/ecc/curves/secp256r1/secp256r1.hpp" -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/serialize/test_helper.hpp" #include @@ -9,7 +9,7 @@ using namespace barretenberg; TEST(ecdsa, msgpack) { - auto [actual, expected] = msgpack_roundtrip(crypto::ecdsa::signature{{}, {}, {}}); + auto [actual, expected] = msgpack_roundtrip(crypto::ecdsa::signature{ {}, {}, {} }); EXPECT_EQ(actual, expected); } diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa_impl.hpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa_impl.hpp index 251b350bc1..33cec9c433 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa_impl.hpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa_impl.hpp @@ -1,7 +1,7 @@ #pragma once #include "barretenberg/numeric/uint256/uint256.hpp" -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "../hmac/hmac.hpp" namespace crypto { diff --git a/cpp/src/barretenberg/crypto/hmac/hmac.hpp b/cpp/src/barretenberg/crypto/hmac/hmac.hpp index 6652626779..e78ef5206c 100644 --- a/cpp/src/barretenberg/crypto/hmac/hmac.hpp +++ b/cpp/src/barretenberg/crypto/hmac/hmac.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include #include #include diff --git a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp index 1840e11894..f503481ce7 100644 --- a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp +++ b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.cpp @@ -1,7 +1,7 @@ #include "c_bind.hpp" #include "pedersen.hpp" #include "pedersen_lookup.hpp" -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/common/timer.hpp" #include "barretenberg/common/mem.hpp" #include "barretenberg/common/streams.hpp" @@ -30,7 +30,6 @@ WASM_EXPORT void pedersen_plookup_compress_fields(uint8_t const* left, uint8_t c barretenberg::fr::serialize_to_buffer(r, result); } - WASM_EXPORT void pedersen__compress(uint8_t const* inputs_buffer, uint8_t* output) { std::vector to_compress; diff --git a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp index 38f57fe899..59b4ef6c40 100644 --- a/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp +++ b/cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/common/timer.hpp" #include "barretenberg/common/mem.hpp" #include "barretenberg/common/streams.hpp" diff --git a/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp b/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp index a3db4cc67c..ffffc314dc 100644 --- a/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp +++ b/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp @@ -1,7 +1,7 @@ #include "c_bind.hpp" #include "pedersen.hpp" #include "pedersen_lookup.hpp" -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/common/timer.hpp" #include "barretenberg/common/mem.hpp" #include "barretenberg/common/streams.hpp" diff --git a/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp b/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp index b7df93f0a4..cd2804b26c 100644 --- a/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp +++ b/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/common/timer.hpp" #include "barretenberg/common/mem.hpp" #include "barretenberg/common/streams.hpp" diff --git a/cpp/src/barretenberg/crypto/schnorr/proof_of_possession.hpp b/cpp/src/barretenberg/crypto/schnorr/proof_of_possession.hpp index 087e3ff37a..69efb5cb8c 100644 --- a/cpp/src/barretenberg/crypto/schnorr/proof_of_possession.hpp +++ b/cpp/src/barretenberg/crypto/schnorr/proof_of_possession.hpp @@ -2,7 +2,7 @@ #include -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "schnorr.hpp" namespace crypto::schnorr { diff --git a/cpp/src/barretenberg/crypto/schnorr/schnorr.hpp b/cpp/src/barretenberg/crypto/schnorr/schnorr.hpp index c91fe20d77..ccd88dceb7 100644 --- a/cpp/src/barretenberg/crypto/schnorr/schnorr.hpp +++ b/cpp/src/barretenberg/crypto/schnorr/schnorr.hpp @@ -8,7 +8,7 @@ #include "barretenberg/crypto/hashers/hashers.hpp" -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/common/streams.hpp" namespace crypto { diff --git a/cpp/src/barretenberg/dsl/acir_format/range_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/range_constraint.hpp index af769b8e06..7f5a512ce6 100644 --- a/cpp/src/barretenberg/dsl/acir_format/range_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/range_constraint.hpp @@ -1,6 +1,6 @@ #pragma once #include -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" namespace acir_format { diff --git a/cpp/src/barretenberg/ecc/fields/field.hpp b/cpp/src/barretenberg/ecc/fields/field.hpp index 3ebaecd5db..05d201c479 100644 --- a/cpp/src/barretenberg/ecc/fields/field.hpp +++ b/cpp/src/barretenberg/ecc/fields/field.hpp @@ -2,7 +2,7 @@ #include #include "barretenberg/common/assert.hpp" #include "barretenberg/common/inline.hpp" -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include #include #include "barretenberg/numeric/random/engine.hpp" diff --git a/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp b/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp index 7da7c62517..c4f7190c5b 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp @@ -4,7 +4,7 @@ #include "barretenberg/ecc/curves/secp256r1/secp256r1.hpp" #include "barretenberg/common/test.hpp" #include -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/serialize/test_helper.hpp" namespace test_affine_element { @@ -126,7 +126,7 @@ TEST(affine_element, infinity_ordering_regression) TEST(affine_element, msgpack) { - auto [actual, expected] = msgpack_roundtrip(secp256k1::g1::affine_element {}); + auto [actual, expected] = msgpack_roundtrip(secp256k1::g1::affine_element{}); EXPECT_EQ(actual, expected); } } // namespace test_affine_element diff --git a/cpp/src/barretenberg/honk/sumcheck/polynomials/univariate.hpp b/cpp/src/barretenberg/honk/sumcheck/polynomials/univariate.hpp index a502aeefb2..a3cc9cf82f 100644 --- a/cpp/src/barretenberg/honk/sumcheck/polynomials/univariate.hpp +++ b/cpp/src/barretenberg/honk/sumcheck/polynomials/univariate.hpp @@ -3,7 +3,7 @@ #include #include #include -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/common/assert.hpp" namespace proof_system::honk::sumcheck { diff --git a/cpp/src/barretenberg/honk/sumcheck/sumcheck.hpp b/cpp/src/barretenberg/honk/sumcheck/sumcheck.hpp index 40c905f29a..8968ee1260 100644 --- a/cpp/src/barretenberg/honk/sumcheck/sumcheck.hpp +++ b/cpp/src/barretenberg/honk/sumcheck/sumcheck.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include #include "barretenberg/honk/sumcheck/relations/relation.hpp" #include "barretenberg/honk/transcript/transcript.hpp" diff --git a/cpp/src/barretenberg/honk/transcript/transcript.hpp b/cpp/src/barretenberg/honk/transcript/transcript.hpp index fe7b078200..fec507a642 100644 --- a/cpp/src/barretenberg/honk/transcript/transcript.hpp +++ b/cpp/src/barretenberg/honk/transcript/transcript.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/crypto/blake3s/blake3s.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/bridge_call_data.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/bridge_call_data.hpp index 5f93f98334..22d79996ad 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/bridge_call_data.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/bridge_call_data.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "../constants.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note.hpp index 67f573ab09..a89e12bcfb 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "create_partial_commitment.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note_tx_data.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note_tx_data.hpp index 5e545f10e6..83c2f6375c 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note_tx_data.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/claim_note_tx_data.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "../bridge_call_data.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/complete_partial_commitment.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/complete_partial_commitment.hpp index dd37fd1b31..27407bf567 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/complete_partial_commitment.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/complete_partial_commitment.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "../../constants.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/compute_nullifier.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/compute_nullifier.hpp index acd08bc934..513b8bfe2e 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/compute_nullifier.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/claim/compute_nullifier.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "../../constants.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/create_partial_commitment.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/create_partial_commitment.hpp index a5c941a7ea..5105074c02 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/create_partial_commitment.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/create_partial_commitment.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "../../constants.hpp" diff --git a/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/value_note.hpp b/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/value_note.hpp index df4367ac49..421106004d 100644 --- a/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/value_note.hpp +++ b/cpp/src/barretenberg/join_split_example/proofs/notes/native/value/value_note.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include "create_partial_commitment.hpp" diff --git a/cpp/src/barretenberg/numeric/uint128/uint128.hpp b/cpp/src/barretenberg/numeric/uint128/uint128.hpp index 27ba089613..884c73e9c6 100644 --- a/cpp/src/barretenberg/numeric/uint128/uint128.hpp +++ b/cpp/src/barretenberg/numeric/uint128/uint128.hpp @@ -4,7 +4,7 @@ #ifdef __i386__ #include -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" namespace numeric { diff --git a/cpp/src/barretenberg/numeric/uint256/uint256.hpp b/cpp/src/barretenberg/numeric/uint256/uint256.hpp index 1c0d8f1cde..52c6411e1c 100644 --- a/cpp/src/barretenberg/numeric/uint256/uint256.hpp +++ b/cpp/src/barretenberg/numeric/uint256/uint256.hpp @@ -15,7 +15,7 @@ #include #include #include -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "../uint128/uint128.hpp" namespace numeric { diff --git a/cpp/src/barretenberg/plonk/proof_system/proving_key/serialize.hpp b/cpp/src/barretenberg/plonk/proof_system/proving_key/serialize.hpp index 67d7c89066..6bc7e79ed5 100644 --- a/cpp/src/barretenberg/plonk/proof_system/proving_key/serialize.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/proving_key/serialize.hpp @@ -2,7 +2,7 @@ #include "proving_key.hpp" #include "barretenberg/polynomials/serialize.hpp" #include "barretenberg/common/throw_or_abort.hpp" -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" namespace proof_system::plonk { diff --git a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp index 3c7606d797..5beee44496 100644 --- a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp @@ -3,19 +3,15 @@ #include #include #include -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/serialize/msgpack.hpp" namespace proof_system::plonk { struct proof { std::vector proof_data; - void msgpack_pack(auto& packer) const { - packer.pack(proof_data); - } - void msgpack_unpack(auto object) { - proof_data = (std::vector)object; - } + void msgpack_pack(auto& packer) const { packer.pack(proof_data); } + void msgpack_unpack(auto object) { proof_data = (std::vector)object; } bool operator==(proof const& other) const = default; }; diff --git a/cpp/src/barretenberg/proof_system/arithmetization/gate_data.hpp b/cpp/src/barretenberg/proof_system/arithmetization/gate_data.hpp index 70740cbe1b..5175867bfd 100644 --- a/cpp/src/barretenberg/proof_system/arithmetization/gate_data.hpp +++ b/cpp/src/barretenberg/proof_system/arithmetization/gate_data.hpp @@ -1,7 +1,7 @@ #pragma once #include #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/serialize/legacy_serialize.hpp" +#include "barretenberg/common/serialize.hpp" namespace proof_system { struct add_triple { From 4ad083eff81647f1ea8de27a2bbf676dc6044451 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 09:45:14 -0400 Subject: [PATCH 68/92] Try to make circuits test work again --- cpp/src/barretenberg/serialize/cbind.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cpp/src/barretenberg/serialize/cbind.hpp b/cpp/src/barretenberg/serialize/cbind.hpp index da25c49d76..f0a12f5818 100644 --- a/cpp/src/barretenberg/serialize/cbind.hpp +++ b/cpp/src/barretenberg/serialize/cbind.hpp @@ -112,7 +112,7 @@ inline void cbind_impl( // returns a C-style string json of the schema inline void cbind_schema_impl(auto func, uint8_t** output_out, size_t* output_len_out) { - (void)func; // unused except for type + (void)func; // unused except for type // Object representation of the cbind auto cbind_obj = get_func_traits(); std::string schema = msgpack::schema_to_string(cbind_obj); @@ -120,8 +120,7 @@ inline void cbind_schema_impl(auto func, uint8_t** output_out, size_t* output_le memcpy(*output_out, schema.c_str(), schema.size() + 1); *output_len_out = schema.size(); } -} // namespace msgpack - +} // namespace msgpack // The CBIND macro is a convenient utility that abstracts away several steps in binding C functions with msgpack // serialization. It creates two separate functions: @@ -136,4 +135,4 @@ inline void cbind_schema_impl(auto func, uint8_t** output_out, size_t* output_le WASM_EXPORT void cname##__schema(uint8_t** output_out, size_t* output_len_out) \ { \ msgpack::cbind_schema_impl(func, output_out, output_len_out); \ - } \ No newline at end of file + } From cec5613c3a55c3283d37fe0ab9229e172e2b15ea Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 12:44:53 -0400 Subject: [PATCH 69/92] fix: initialization warning --- .../serialize/msgpack_schema.test.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp b/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp index 4d8d5476d1..244cd55f94 100644 --- a/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp +++ b/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp @@ -7,33 +7,33 @@ // TODO eventually move to barretenberg struct GoodExample { - barretenberg::fr a; - barretenberg::fr b; + barretenberg::fr a = 0; + barretenberg::fr b = 0; MSGPACK(a, b); } good_example; struct BadExampleOverlap { - barretenberg::fr a; - barretenberg::fr b; + barretenberg::fr a = 0; + barretenberg::fr b = 0; MSGPACK(a, a); } bad_example_overlap; struct BadExampleIncomplete { - barretenberg::fr a; - barretenberg::fr b; + barretenberg::fr a = 0; + barretenberg::fr b = 0; MSGPACK(a); } bad_example_incomplete; struct BadExampleCompileTimeError { std::vector a; - barretenberg::fr b; + barretenberg::fr b = 0; MSGPACK(b); // Type mismatch, expect 'a', will catch at compile-time } bad_example_compile_time_error; struct BadExampleOutOfObject { - barretenberg::fr a; - barretenberg::fr b; + barretenberg::fr a = 0; + barretenberg::fr b = 0; void msgpack(auto ar) { BadExampleOutOfObject other_object; @@ -68,7 +68,7 @@ TEST(msgpack_tests, msgpack_sanity_sanity) struct ComplicatedSchema { std::vector> array; std::optional good_or_not; - barretenberg::fr bare; + barretenberg::fr bare = 0; std::variant huh; MSGPACK(array, good_or_not, bare, huh); } complicated_schema; From 930b2831caeb4f9a4186ef4426e17f6ea9dd7ae6 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 15:03:09 -0400 Subject: [PATCH 70/92] fix: prefer default constructor for field, related cleanup --- .../barretenberg/crypto/ecdsa/ecdsa.test.cpp | 2 +- cpp/src/barretenberg/ecc/fields/field.hpp | 10 +++++++--- .../ecc/groups/affine_element.test.cpp | 2 +- .../barretenberg/ecc/groups/element_impl.hpp | 3 --- .../turbo_arithmetic_widget.hpp | 1 - .../turbo_circuit_constructor.cpp | 7 +------ .../serialize/msgpack_schema.test.cpp | 20 +++++++++---------- 7 files changed, 20 insertions(+), 25 deletions(-) diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp index fcab380014..5d2781571a 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.test.cpp @@ -9,7 +9,7 @@ using namespace barretenberg; TEST(ecdsa, msgpack) { - auto [actual, expected] = msgpack_roundtrip(crypto::ecdsa::signature{ {}, {}, {} }); + auto [actual, expected] = msgpack_roundtrip(crypto::ecdsa::signature{}); EXPECT_EQ(actual, expected); } diff --git a/cpp/src/barretenberg/ecc/fields/field.hpp b/cpp/src/barretenberg/ecc/fields/field.hpp index 05d201c479..4bd9a94d9b 100644 --- a/cpp/src/barretenberg/ecc/fields/field.hpp +++ b/cpp/src/barretenberg/ecc/fields/field.hpp @@ -24,12 +24,16 @@ namespace barretenberg { template struct alignas(32) field { public: - // We don't initialize data by default since we'd lose a lot of time on pointless initializations. + // We don't initialize data in the default constructor since we'd lose a lot of time on huge array initializations. // Other alternatives have been noted, such as casting to get around constructors where they matter, // however it is felt that sanitizer tools (e.g. MSAN) can detect garbage well, whereas doing // hacky casts where needed would require rework to critical algos like MSM, FFT, Sumcheck. - // Instead, the recommended solution is use an explicit = 0 where initialization is important. - field() noexcept {} + // Instead, the recommended solution is use an explicit {} where initialization is important: + // field f; // not initialized + // field f{}; // zero-initialized + // std::array arr; // not initialized, good for huge N + // std::array arr {}; // zero-initialized, preferable for moderate N + field() = default; constexpr field(const uint256_t& input) noexcept : data{ input.data[0], input.data[1], input.data[2], input.data[3] } diff --git a/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp b/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp index c4f7190c5b..352cfe2351 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element.test.cpp @@ -126,7 +126,7 @@ TEST(affine_element, infinity_ordering_regression) TEST(affine_element, msgpack) { - auto [actual, expected] = msgpack_roundtrip(secp256k1::g1::affine_element{}); + auto [actual, expected] = msgpack_roundtrip(secp256k1::g1::affine_element{ 1, 1 }); EXPECT_EQ(actual, expected); } } // namespace test_affine_element diff --git a/cpp/src/barretenberg/ecc/groups/element_impl.hpp b/cpp/src/barretenberg/ecc/groups/element_impl.hpp index c3a7859530..60f2faef28 100644 --- a/cpp/src/barretenberg/ecc/groups/element_impl.hpp +++ b/cpp/src/barretenberg/ecc/groups/element_impl.hpp @@ -925,7 +925,6 @@ element element::random_coordinates_on_curve(numeric::rand Fq yy; Fq x; Fq y; - Fq t0; while (!found_one) { x = Fq::random_element(engine); yy = x.sqr() * x + T::b; @@ -935,8 +934,6 @@ element element::random_coordinates_on_curve(numeric::rand auto [found_root, y1] = yy.sqrt(); y = y1; found_one = found_root; - // t0 = y.sqr(); - // found_one = (yy == t0); } return { x, y, Fq::one() }; } diff --git a/cpp/src/barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_arithmetic_widget.hpp b/cpp/src/barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_arithmetic_widget.hpp index 9daaba1f0c..5b46fb8b80 100644 --- a/cpp/src/barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_arithmetic_widget.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_arithmetic_widget.hpp @@ -156,7 +156,6 @@ template class TurboArithme Field T3; Field T4; Field T5; - Field T6; /** * Quad extraction term. This term is only active when q_arith is set to 2. diff --git a/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp b/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp index 517587f750..8bc4b461e2 100644 --- a/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp +++ b/cpp/src/barretenberg/proof_system/circuit_constructors/turbo_circuit_constructor.cpp @@ -929,14 +929,9 @@ inline bool TurboCircuitConstructor::lazy_logic_gate_check(const size_t gate_ind constexpr fr three(3); constexpr fr minus_one = -fr::one(); - fr delta_sum; - fr delta_squared_sum; fr T0; fr T1; fr T2; - fr T3; - fr T4; - fr identity; // T0 = a T0 = wire_1_value + wire_1_value; @@ -1448,7 +1443,7 @@ inline fr TurboCircuitConstructor::fixed_base_gate_evaluation(const size_t gate_ * */ bool TurboCircuitConstructor::check_circuit() { -//#define LAZY_CIRCUIT_CHECKS +// #define LAZY_CIRCUIT_CHECKS #ifdef LAZY_CIRCUIT_CHECKS for (size_t i = 0; i < num_gates; i++) { if (!q_arith[i].is_zero() && !lazy_arithmetic_gate_check(i)) { diff --git a/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp b/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp index 244cd55f94..4d8d5476d1 100644 --- a/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp +++ b/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp @@ -7,33 +7,33 @@ // TODO eventually move to barretenberg struct GoodExample { - barretenberg::fr a = 0; - barretenberg::fr b = 0; + barretenberg::fr a; + barretenberg::fr b; MSGPACK(a, b); } good_example; struct BadExampleOverlap { - barretenberg::fr a = 0; - barretenberg::fr b = 0; + barretenberg::fr a; + barretenberg::fr b; MSGPACK(a, a); } bad_example_overlap; struct BadExampleIncomplete { - barretenberg::fr a = 0; - barretenberg::fr b = 0; + barretenberg::fr a; + barretenberg::fr b; MSGPACK(a); } bad_example_incomplete; struct BadExampleCompileTimeError { std::vector a; - barretenberg::fr b = 0; + barretenberg::fr b; MSGPACK(b); // Type mismatch, expect 'a', will catch at compile-time } bad_example_compile_time_error; struct BadExampleOutOfObject { - barretenberg::fr a = 0; - barretenberg::fr b = 0; + barretenberg::fr a; + barretenberg::fr b; void msgpack(auto ar) { BadExampleOutOfObject other_object; @@ -68,7 +68,7 @@ TEST(msgpack_tests, msgpack_sanity_sanity) struct ComplicatedSchema { std::vector> array; std::optional good_or_not; - barretenberg::fr bare = 0; + barretenberg::fr bare; std::variant huh; MSGPACK(array, good_or_not, bare, huh); } complicated_schema; From e9c4d6edfa5ad589a4c6ab2c787dfb22124b6f0e Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 17:58:48 -0400 Subject: [PATCH 71/92] Grand rename --- cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp | 2 +- .../dsl/acir_format/logic_constraint.hpp | 2 +- .../dsl/acir_format/sha256_constraint.hpp | 4 +- cpp/src/barretenberg/ecc/curves/bn254/fq.hpp | 4 +- cpp/src/barretenberg/ecc/curves/bn254/g1.hpp | 5 +- .../ecc/groups/affine_element.hpp | 2 +- cpp/src/barretenberg/ecc/groups/element.hpp | 2 +- .../honk/sumcheck/sumcheck.test.cpp | 1 - .../plonk/proof_system/prover/c_bind.cpp | 6 ++ .../plonk/proof_system/types/proof.hpp | 4 +- .../verification_key/verification_key.hpp | 49 +++++----- cpp/src/barretenberg/serialize/cbind.hpp | 72 ++++---------- .../serialize/{cbind_fwd.h => cbind_fwd.hpp} | 6 +- cpp/src/barretenberg/serialize/msgpack.hpp | 2 +- .../msgpack_impl/check_memory_span.hpp | 10 +- .../serialize/msgpack_impl/concepts.hpp | 10 +- .../serialize/msgpack_impl/func_traits.hpp | 40 ++++++++ .../serialize/msgpack_impl/schema_impl.hpp | 95 ++++++++++--------- .../serialize/msgpack_impl/schema_name.hpp | 4 +- .../msgpack_impl/struct_map_impl.hpp | 18 ++-- .../serialize/msgpack_schema.test.cpp | 16 ++-- .../barretenberg/serialize/raw_pointer.hpp | 24 ++--- .../barretenberg/serialize/test_helper.hpp | 29 ++++-- .../stdlib/primitives/address/address.hpp | 4 +- .../native_aggregation_state.hpp | 2 +- cpp/src/msgpack-c | 2 +- 26 files changed, 214 insertions(+), 201 deletions(-) rename cpp/src/barretenberg/serialize/{cbind_fwd.h => cbind_fwd.hpp} (56%) create mode 100644 cpp/src/barretenberg/serialize/msgpack_impl/func_traits.hpp diff --git a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp index 4ceb5ef6dd..df1e887fdc 100644 --- a/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp +++ b/cpp/src/barretenberg/crypto/ecdsa/ecdsa.hpp @@ -17,7 +17,7 @@ struct signature { std::array s; uint8_t v; // for serialization, update with any new fields - MSGPACK(r, s, v); + MSGPACK_FIELDS(r, s, v); }; template diff --git a/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp index d24f57bd91..d2ed943398 100644 --- a/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/logic_constraint.hpp @@ -14,7 +14,7 @@ struct LogicConstraint { friend bool operator==(LogicConstraint const& lhs, LogicConstraint const& rhs) = default; // for serialization, update with any new fields - MSGPACK(a, b, result, num_bits, is_xor_gate); + MSGPACK_FIELDS(a, b, result, num_bits, is_xor_gate); }; void create_logic_gate(Composer& composer, uint32_t a, uint32_t b, uint32_t result, size_t num_bits, bool is_xor_gate); diff --git a/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp b/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp index 9e0b61129d..3c434bae26 100644 --- a/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp +++ b/cpp/src/barretenberg/dsl/acir_format/sha256_constraint.hpp @@ -11,7 +11,7 @@ struct Sha256Input { friend bool operator==(Sha256Input const& lhs, Sha256Input const& rhs) = default; // for serialization, update with any new fields - MSGPACK(witness, num_bits); + MSGPACK_FIELDS(witness, num_bits); }; struct Sha256Constraint { @@ -20,7 +20,7 @@ struct Sha256Constraint { friend bool operator==(Sha256Constraint const& lhs, Sha256Constraint const& rhs) = default; // for serialization, update with any new fields - MSGPACK(inputs, result); + MSGPACK_FIELDS(inputs, result); }; // This function does not work (properly) because the stdlib:sha256 function is not working correctly for 512 bits diff --git a/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp b/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp index 83e29fd692..080184d2db 100644 --- a/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp +++ b/cpp/src/barretenberg/ecc/curves/bn254/fq.hpp @@ -62,10 +62,8 @@ typedef field fq; } // namespace barretenberg -namespace msgpack { // define this as a named alias in msgpack schema generation -inline void schema_pack(auto& packer, barretenberg::fq const&) +inline void msgpack_schema_pack(auto& packer, barretenberg::fq const&) { packer.pack_alias("Fq", "bin32"); } -} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp b/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp index 17213910e4..6d58fd1836 100644 --- a/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp +++ b/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp @@ -20,9 +20,8 @@ typedef group g1; } // namespace barretenberg -namespace msgpack { // specialize the name in msgpack schema generation -inline std::string schema_name(barretenberg::g1::affine_element const &) { +inline std::string msgpack_schema_name(barretenberg::g1::affine_element const&) +{ return "G1AffineElement"; } -} \ No newline at end of file diff --git a/cpp/src/barretenberg/ecc/groups/affine_element.hpp b/cpp/src/barretenberg/ecc/groups/affine_element.hpp index 3c95613ef3..f41787661c 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element.hpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element.hpp @@ -178,7 +178,7 @@ template class alignas(64) affine_el Fq x; Fq y; // for serialization: update up with new fields - MSGPACK(x, y); + MSGPACK_FIELDS(x, y); }; template void read(B& it, affine_element& value) diff --git a/cpp/src/barretenberg/ecc/groups/element.hpp b/cpp/src/barretenberg/ecc/groups/element.hpp index a8845f881d..430db6abf7 100644 --- a/cpp/src/barretenberg/ecc/groups/element.hpp +++ b/cpp/src/barretenberg/ecc/groups/element.hpp @@ -124,7 +124,7 @@ template class alignas(32) element { // return { x, y, Fq::one() }; // } // for serialization: update up with new fields - MSGPACK(x, y, z); + MSGPACK_FIELDS(x, y, z); static void conditional_negate_affine(const affine_element& in, affine_element& out, diff --git a/cpp/src/barretenberg/honk/sumcheck/sumcheck.test.cpp b/cpp/src/barretenberg/honk/sumcheck/sumcheck.test.cpp index bae0506199..356ad662b9 100644 --- a/cpp/src/barretenberg/honk/sumcheck/sumcheck.test.cpp +++ b/cpp/src/barretenberg/honk/sumcheck/sumcheck.test.cpp @@ -96,7 +96,6 @@ TEST(Sumcheck, PolynomialNormalization) std::array id_3; std::array lagrange_first; std::array lagrange_last; - std::array pow_zeta; for (size_t i = 0; i < multivariate_n; i++) { w_l[i] = FF::random_element(); w_r[i] = FF::random_element(); diff --git a/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp b/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp index 2b05d87bd2..df6b1b03e7 100644 --- a/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp +++ b/cpp/src/barretenberg/plonk/proof_system/prover/c_bind.cpp @@ -8,6 +8,10 @@ using namespace barretenberg; +// TODO(AD): This __wasm__ guard is a hack, but these test functions are currently +// the only consumer of this API and it has no native definition. +// Eventually, remove this, and don't rely on asyncify with Charlie's work. +#ifdef __wasm__ /** * Called by `barretenberg_wasm.test.ts` to test the asyncify intrumentation and logic that * allows for WASM code to make calls to async code in JS. @@ -30,6 +34,8 @@ WASM_EXPORT void* test_async_func(size_t size, int val) return addr; } } +#endif + /** * @brief Simple wrapper for env_load_verifier_crs. * @return The CRS. diff --git a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp index 5beee44496..389bf211e2 100644 --- a/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/types/proof.hpp @@ -46,10 +46,8 @@ inline std::ostream& operator<<(std::ostream& os, proof const& data) } // namespace proof_system::plonk -namespace msgpack { // help our msgpack schema compiler with this typedef -inline void schema_pack(auto& packer, proof_system::plonk::proof const&) +inline void msgpack_schema_pack(auto& packer, proof_system::plonk::proof const&) { packer.pack_alias("Proof", "bin32"); } -} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp index a04e5af793..fb1aceb828 100644 --- a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp @@ -20,14 +20,12 @@ struct verification_key_data { std::vector recursive_proof_public_input_indices; // for serialization: update with any new fields - MSGPACK( - composer_type, - circuit_size, - num_public_inputs, - commitments, - contains_recursive_proof, - recursive_proof_public_input_indices - ); + MSGPACK_FIELDS(composer_type, + circuit_size, + num_public_inputs, + commitments, + contains_recursive_proof, + recursive_proof_public_input_indices); barretenberg::fr compress_native(size_t const hash_index = 0); }; @@ -97,29 +95,29 @@ struct verification_key { size_t program_width = 3; // for serialization: update up with new fields - void msgpack_pack(auto& packer) const { - verification_key_data data = { - composer_type, - static_cast(circuit_size), - static_cast(num_public_inputs), - commitments, - contains_recursive_proof, - recursive_proof_public_input_indices - }; + void msgpack_pack(auto& packer) const + { + verification_key_data data = { composer_type, + static_cast(circuit_size), + static_cast(num_public_inputs), + commitments, + contains_recursive_proof, + recursive_proof_public_input_indices }; packer.pack(data); } - void msgpack_unpack(auto obj) { + void msgpack_unpack(auto obj) + { verification_key_data data = obj; auto env_crs = std::make_unique(); - *this = verification_key {std::move(data), env_crs->get_verifier_crs()}; + *this = verification_key{ std::move(data), env_crs->get_verifier_crs() }; } }; -namespace msgpack { -inline void msgpack_schema(auto &packer, proof_system::plonk::verification_key const &) { +// specialize schema serialization +inline void msgpack_schema(auto& packer, proof_system::plonk::verification_key const&) +{ packer.pack_schema(proof_system::plonk::verification_key_data{}); } -} // namespace msgpack template inline void read(B& buf, verification_key& key) { @@ -161,3 +159,10 @@ inline std::ostream& operator<<(std::ostream& os, verification_key const& key) }; } // namespace proof_system::plonk + +// help our msgpack schema compiler with this struct +// Alias verification_key as verification_key_data +inline void msgpack_schema_pack(auto& packer, proof_system::plonk::verification_key const&) +{ + msgpack_schema_pack(packer, proof_system::plonk::verification_key_data{}); +} \ No newline at end of file diff --git a/cpp/src/barretenberg/serialize/cbind.hpp b/cpp/src/barretenberg/serialize/cbind.hpp index f0a12f5818..9e889dda18 100644 --- a/cpp/src/barretenberg/serialize/cbind.hpp +++ b/cpp/src/barretenberg/serialize/cbind.hpp @@ -13,12 +13,17 @@ #include "msgpack_impl/schema_name.hpp" #include "msgpack_impl/struct_map_impl.hpp" #include "msgpack_impl/variant_impl.hpp" +#include "msgpack_impl/func_traits.hpp" #include #include -namespace msgpack { -inline std::pair encode_buffer(auto&& obj) +/** + * Represents this as a bbmalloc'ed object, fit for sending to e.g. TypeScript. + * @param obj The object. + * @return The buffer pointer/size pair. + */ +inline std::pair msgpack_encode_buffer(auto&& obj) { // Create a buffer to store the encoded data msgpack::sbuffer buffer; @@ -30,13 +35,7 @@ inline std::pair encode_buffer(auto&& obj) return { output, buffer.size() }; } -template inline void decode(T* value, const uint8_t* encoded_data, size_t encoded_data_size) -{ - // Create a MsgPack unpacker - msgpack::unpack((const char*)encoded_data, encoded_data_size).get().convert(*value); -} - -std::string string_encode(const auto& obj) +std::string msgpack_json_encode(const auto& obj) { std::stringstream output; msgpack::pack(output, obj); @@ -46,7 +45,8 @@ std::string string_encode(const auto& obj) pretty_output << oh.get() << std::endl; return pretty_output.str(); } -void print(const auto& obj) + +void msgpack_json_print(const auto& obj) { std::stringstream output; msgpack::pack(output, obj); @@ -57,70 +57,32 @@ void print(const auto& obj) logstr(pretty_output.str().c_str()); } -template struct func_traits; - -template struct func_traits { - typedef std::tuple Args; - Args args; - R ret; - MSGPACK(args, ret); -}; -template struct func_traits { - typedef std::tuple Args; - Args args; - R ret; - MSGPACK(args, ret); -}; - -template struct func_traits { - typedef std::tuple Args; - Args args; - R ret; - MSGPACK(args, ret); -}; - -// Template metamagic for figuring out the parameters and return type of a function -template -concept Callable = - requires() { typename std::enable_if_t, void>; }; - -template constexpr auto get_func_traits() -{ - return func_traits(); -} - -template constexpr auto get_func_traits() -{ - return func_traits(); -} - template constexpr auto param_tuple() { return typename decltype(get_func_traits())::Args{}; } -inline void cbind_impl( +inline void msgpack_cbind_impl( auto func, const uint8_t* input_in, size_t input_len_in, uint8_t** output_out, size_t* output_len_out) { auto params = param_tuple(); - msgpack::decode(¶ms, input_in, input_len_in); - auto [output, output_len] = msgpack::encode_buffer(std::apply(func, params)); + msgpack::unpack((const char*)input_in, input_len_in).get().convert(params); + auto [output, output_len] = msgpack_encode_buffer(std::apply(func, params)); *output_out = output; *output_len_out = output_len; } // returns a C-style string json of the schema -inline void cbind_schema_impl(auto func, uint8_t** output_out, size_t* output_len_out) +inline void msgpack_cbind_schema_impl(auto func, uint8_t** output_out, size_t* output_len_out) { (void)func; // unused except for type // Object representation of the cbind auto cbind_obj = get_func_traits(); - std::string schema = msgpack::schema_to_string(cbind_obj); + std::string schema = msgpack_schema_to_string(cbind_obj); *output_out = (uint8_t*)aligned_alloc(64, schema.size() + 1); memcpy(*output_out, schema.c_str(), schema.size() + 1); *output_len_out = schema.size(); } -} // namespace msgpack // The CBIND macro is a convenient utility that abstracts away several steps in binding C functions with msgpack // serialization. It creates two separate functions: @@ -130,9 +92,9 @@ inline void cbind_schema_impl(auto func, uint8_t** output_out, size_t* output_le #define CBIND(cname, func) \ WASM_EXPORT void cname(const uint8_t* input_in, size_t input_len_in, uint8_t** output_out, size_t* output_len_out) \ { \ - msgpack::cbind_impl(func, input_in, input_len_in, output_out, output_len_out); \ + msgpack_cbind_impl(func, input_in, input_len_in, output_out, output_len_out); \ } \ WASM_EXPORT void cname##__schema(uint8_t** output_out, size_t* output_len_out) \ { \ - msgpack::cbind_schema_impl(func, output_out, output_len_out); \ + msgpack_cbind_schema_impl(func, output_out, output_len_out); \ } diff --git a/cpp/src/barretenberg/serialize/cbind_fwd.h b/cpp/src/barretenberg/serialize/cbind_fwd.hpp similarity index 56% rename from cpp/src/barretenberg/serialize/cbind_fwd.h rename to cpp/src/barretenberg/serialize/cbind_fwd.hpp index fef437e4dc..ef5eee9204 100644 --- a/cpp/src/barretenberg/serialize/cbind_fwd.h +++ b/cpp/src/barretenberg/serialize/cbind_fwd.hpp @@ -1,9 +1,11 @@ #pragma once #include #include -// CBIND forward declarations for msgback default bind format (encode as tuple of args and return value as msgpack string) +// CBIND forward declarations for msgback default bind format (encode as tuple of args and return value as msgpack +// string) #define WASM_EXPORT extern "C" __attribute__((visibility("default"))) #define CBIND_DECL(cname) \ - WASM_EXPORT void cname(const uint8_t* input_in, size_t input_len_in, uint8_t** output_out, size_t* output_len_out);\ + WASM_EXPORT void cname( \ + const uint8_t* input_in, size_t input_len_in, uint8_t** output_out, size_t* output_len_out); \ WASM_EXPORT void cname##__schema(uint8_t** output_out, size_t* output_len_out); diff --git a/cpp/src/barretenberg/serialize/msgpack.hpp b/cpp/src/barretenberg/serialize/msgpack.hpp index 8d0d917cfb..6d418ba766 100644 --- a/cpp/src/barretenberg/serialize/msgpack.hpp +++ b/cpp/src/barretenberg/serialize/msgpack.hpp @@ -95,7 +95,7 @@ e.g. unpacking #include "msgpack_impl/concepts.hpp" // Helper for above documented syntax -#define MSGPACK(...) \ +#define MSGPACK_FIELDS(...) \ void msgpack(auto pack_fn) \ { \ pack_fn(NVP(__VA_ARGS__)); \ diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/check_memory_span.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/check_memory_span.hpp index e107b577e4..7a310f7cf3 100644 --- a/cpp/src/barretenberg/serialize/msgpack_impl/check_memory_span.hpp +++ b/cpp/src/barretenberg/serialize/msgpack_impl/check_memory_span.hpp @@ -29,12 +29,12 @@ template std::string check_memory_span(T* obj, Ar // Check if any of the Args* pointers overlap. auto last_end = pointers[i - 1].first + pointers[i - 1].second; if (last_end > pointers[i].first) { - return "Overlap in " + msgpack::schema_name(*obj) + " MSGPACK() params detected!"; + return "Overlap in " + msgpack_schema_name(*obj) + " MSGPACK_FIELDS() params detected!"; } // Check if gap is too large. // Give some fuzzy room in case of 64 byte alignment restrictions. if (__aligned_for(last_end) < pointers[i].first) { - return "Gap in " + msgpack::schema_name(*obj) + " MSGPACK() params detected before member #" + + return "Gap in " + msgpack_schema_name(*obj) + " MSGPACK_FIELDS() params detected before member #" + std::to_string(i) + " !"; } } @@ -43,7 +43,7 @@ template std::string check_memory_span(T* obj, Ar uintptr_t t_start = reinterpret_cast(obj); uintptr_t t_end = t_start + sizeof(T); if (pointers.front().first < t_start || pointers.back().first + pointers.back().second > t_end) { - return "Some " + msgpack::schema_name(*obj) + " MSGPACK() params don't exist in object!"; + return "Some " + msgpack_schema_name(*obj) + " MSGPACK_FIELDS() params don't exist in object!"; } // Check if all of T* memory is used by the Args* pointers. @@ -54,12 +54,12 @@ template std::string check_memory_span(T* obj, Ar } size_t total_size = end - start; if (__aligned_for(total_size) < sizeof(T)) { - return "Incomplete " + msgpack::schema_name(*obj) + " MSGPACK() params! Not all of object specified."; + return "Incomplete " + msgpack_schema_name(*obj) + " MSGPACK_FIELDS() params! Not all of object specified."; } return {}; } -template std::string check_msgpack_method(T& object) +template std::string check_msgpack_method(T& object) { std::string result; auto checker = [&](auto&... values) { result = check_memory_span(&object, &values...); }; diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp index f9add286a7..07579bdf24 100644 --- a/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp +++ b/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp @@ -1,11 +1,11 @@ #pragma once struct DoNothing { - void operator()(auto...){} + void operator()(auto...) {} }; -namespace msgpack { -template +namespace msgpack_concepts { +template concept HasMsgPack = requires(T t, DoNothing nop) { t.msgpack(nop); }; -template +template concept HasMsgPackPack = requires(T t, DoNothing nop) { t.msgpack_pack(nop); }; -} \ No newline at end of file +} // namespace msgpack_concepts \ No newline at end of file diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/func_traits.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/func_traits.hpp new file mode 100644 index 0000000000..5805bc4cea --- /dev/null +++ b/cpp/src/barretenberg/serialize/msgpack_impl/func_traits.hpp @@ -0,0 +1,40 @@ +#pragma once +#include +#include "../msgpack.hpp" + +template struct func_traits; + +template struct func_traits { + typedef std::tuple Args; + Args args; + R ret; + MSGPACK_FIELDS(args, ret); +}; +template struct func_traits { + typedef std::tuple Args; + Args args; + R ret; + MSGPACK_FIELDS(args, ret); +}; + +template struct func_traits { + typedef std::tuple Args; + Args args; + R ret; + MSGPACK_FIELDS(args, ret); +}; + +// Template metamagic for figuring out the parameters and return type of a function +template +concept Callable = + requires() { typename std::enable_if_t, void>; }; + +template constexpr auto get_func_traits() +{ + return func_traits(); +} + +template constexpr auto get_func_traits() +{ + return func_traits(); +} diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp index 08e74d0079..e67e68edee 100644 --- a/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp +++ b/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp @@ -6,14 +6,12 @@ #include #include "schema_name.hpp" -namespace msgpack { - /** * Define a serialization schema based on compile-time information about a type being serialized. * This is then consumed by typescript to make bindings. */ -struct SchemaPacker : packer { - SchemaPacker(msgpack::sbuffer& stream) +struct MsgpackSchemaPacker : msgpack::packer { + MsgpackSchemaPacker(msgpack::sbuffer& stream) : packer(stream) {} // For tracking emitted types @@ -48,24 +46,29 @@ struct SchemaPacker : packer { * @tparam T the object's type. * @param obj the object. */ - template - void pack_schema(const T& obj) - { - schema_pack(*this, obj); - } + template void pack_schema(const T& obj) { msgpack_schema_pack(*this, obj); } }; // Helper for packing (key, value, key, value, ...) arguments -inline void _schema_pack_map_content(SchemaPacker&) +inline void _schema_pack_map_content(MsgpackSchemaPacker&) { // base case } + +namespace msgpack_concepts { +template +concept SchemaPackable = requires(T value, MsgpackSchemaPacker packer) { msgpack_schema_pack(packer, value); }; +} + // Helper for packing (key, value, key, value, ...) arguments template -inline void _schema_pack_map_content(SchemaPacker& packer, std::string key, Value value, Rest... rest) +inline void _schema_pack_map_content(MsgpackSchemaPacker& packer, std::string key, Value value, Rest... rest) { + static_assert( + msgpack_concepts::SchemaPackable, + "see the first type argument in the error trace, it might require a specialization of msgpack_schema_pack"); packer.pack(key); - schema_pack(packer, value); + msgpack_schema_pack(packer, value); _schema_pack_map_content(packer, rest...); } @@ -75,10 +78,10 @@ inline void _schema_pack_map_content(SchemaPacker& packer, std::string key, Valu * @param packer the schema packer. */ template - requires(!HasMsgPack && !HasMsgPackPack) -inline void schema_pack(SchemaPacker& packer, T const&) + requires(!msgpack_concepts::HasMsgPack && !msgpack_concepts::HasMsgPackPack) +inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, T const&) { - packer.pack(schema_name(T{})); + packer.pack(msgpack_schema_name(T{})); } /** @@ -88,9 +91,9 @@ inline void schema_pack(SchemaPacker& packer, T const&) * @param packer Our special packer. * @param object The object in question. */ -template inline void schema_pack(SchemaPacker& packer, T const& object) +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, T const& object) { - std::string type = schema_name(object); + std::string type = msgpack_schema_name(object); if (packer.set_emitted(type)) { packer.pack(type); return; // already emitted @@ -106,53 +109,59 @@ template inline void schema_pack(SchemaPacker& packer, T const& o }); } -// Alias verification_key as verification_key_data -//inline void schema_pack(SchemaPacker& packer, proof_system::plonk::verification_key const&) -//{ -// schema_pack(packer, proof_system::plonk::verification_key_data{}); -//} - // Recurse over any templated containers // Outputs e.g. ['vector', ['sub-type']] -template inline void _schema_pack(SchemaPacker& packer, const std::string& schema_name) +template +inline void _msgpack_schema_pack(MsgpackSchemaPacker& packer, const std::string& schema_name) { packer.pack_array(2); packer.pack(schema_name); packer.pack_array(sizeof...(Args)); - (schema_pack(packer, Args{}), ...); /* pack schemas of all template Args */ + // helper for better errors + auto pack = [&](auto arg) { + static_assert(msgpack_concepts::SchemaPackable, + "see the type argument of this lambda in the error trace, it might require a specialization of " + "msgpack_schema_pack"); + msgpack_schema_pack(packer, arg); + }; + + // Note: if this fails to compile, check first in list of template Arg's + // it may need a msgpack_schema_pack specialization (particularly if it doesn't define MSGPACK_FIELDS). + (pack(Args{}), ...); /* pack schemas of all template Args */ } -template inline void schema_pack(SchemaPacker& packer, std::tuple const&) +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::tuple const&) { - _schema_pack(packer, "tuple"); + _msgpack_schema_pack(packer, "tuple"); } -template inline void schema_pack(SchemaPacker& packer, std::map const&) +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::map const&) { - _schema_pack(packer, "map"); + _msgpack_schema_pack(packer, "map"); } -template inline void schema_pack(SchemaPacker& packer, std::optional const&) +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::optional const&) { - _schema_pack(packer, "optional"); + _msgpack_schema_pack(packer, "optional"); } -template inline void schema_pack(SchemaPacker& packer, std::vector const&) +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::vector const&) { - _schema_pack(packer, "vector"); + _msgpack_schema_pack(packer, "vector"); } -template inline void schema_pack(SchemaPacker& packer, std::variant const&) +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::variant const&) { - _schema_pack(packer, "variant"); + _msgpack_schema_pack(packer, "variant"); } -template inline void schema_pack(SchemaPacker& packer, std::shared_ptr const&) +template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::shared_ptr const&) { - _schema_pack(packer, "shared_ptr"); + _msgpack_schema_pack(packer, "shared_ptr"); } // Outputs e.g. ['array', ['array-type', 'N']] -template inline void schema_pack(SchemaPacker& packer, std::array const&) +template +inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::array const&) { packer.pack_array(2); packer.pack("array"); packer.pack_array(2); /* param list format for consistency*/ - schema_pack(packer, T{}); + msgpack_schema_pack(packer, T{}); packer.pack(N); } @@ -162,15 +171,13 @@ template inline void schema_pack(SchemaPacker& packe * @param obj The object to print schema of. * @return std::string The schema as a string. */ -std::string schema_to_string(auto obj) +std::string msgpack_schema_to_string(auto obj) { msgpack::sbuffer output; - SchemaPacker printer{ output }; - schema_pack(printer, obj); + MsgpackSchemaPacker printer{ output }; + msgpack_schema_pack(printer, obj); msgpack::object_handle oh = msgpack::unpack(output.data(), output.size()); std::stringstream pretty_output; pretty_output << oh.get() << std::endl; return pretty_output.str(); } - -} // namespace msgpack diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/schema_name.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/schema_name.hpp index 7c1f46961e..c70c8f2a5b 100644 --- a/cpp/src/barretenberg/serialize/msgpack_impl/schema_name.hpp +++ b/cpp/src/barretenberg/serialize/msgpack_impl/schema_name.hpp @@ -3,7 +3,6 @@ #include #include -namespace msgpack { /** * Converts C++ type information into a human-readable format. * This function leverages __cxa_demangle to demangle the name generated by typeid. @@ -12,7 +11,7 @@ namespace msgpack { * @tparam T the type. * @return the readable schema name. */ -template std::string schema_name(T const&) +template std::string msgpack_schema_name(T const&) { std::string result = abi::__cxa_demangle(typeid(T).name(), NULL, NULL, NULL); if (result.find("basic_string") != std::string::npos) { @@ -30,4 +29,3 @@ template std::string schema_name(T const&) } return result; } -} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp index 81dd27ab69..4bdf4909c9 100644 --- a/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp +++ b/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp @@ -33,15 +33,15 @@ concept MsgpackConstructible = requires(T object, Args... args) { T{ args... }; namespace msgpack::adaptor { // reads structs with msgpack() method from a JSON-like dictionary -template struct convert { +template struct convert { msgpack::object const& operator()(msgpack::object const& o, T& v) const { static_assert(std::is_default_constructible_v, - "MSGPACK requires default-constructible types (used during unpacking)"); + "MSGPACK_FIELDS requires default-constructible types (used during unpacking)"); v.msgpack([&](auto&... args) { auto static_checker = [&](auto&... value_args) { static_assert(msgpack::MsgpackConstructible, - "MSGPACK requires a constructor that can take the types listed in MSGPACK. " + "MSGPACK_FIELDS requires a constructor that can take the types listed in MSGPACK_FIELDS. " "Type or arg count mismatch, or member initializer constructor not available."); }; std::apply(static_checker, drop_keys(std::tie(args...))); @@ -52,17 +52,19 @@ template struct convert { }; // converts structs with msgpack() method from a JSON-like dictionary -template struct pack { +template struct pack { template packer& operator()(msgpack::packer& o, T const& v) const { static_assert(std::is_default_constructible_v, - "MSGPACK requires default-constructible types (used during unpacking)"); + "MSGPACK_FIELDS requires default-constructible types (used during unpacking)"); const_cast(v).msgpack([&](auto&... args) { auto static_checker = [&](auto&... value_args) { static_assert(msgpack::MsgpackConstructible, - "T requires a constructor that can take the fields listed in MSGPACK (T will be in template parameters in the compiler stack trace)" - "Check the MSGPACK macro usage in T for incompleteness or wrong order." - "Alternatively, a matching member initializer constructor might not be available for T and should be defined."); + "T requires a constructor that can take the fields listed in MSGPACK_FIELDS (T will be " + "in template parameters in the compiler stack trace)" + "Check the MSGPACK_FIELDS macro usage in T for incompleteness or wrong order." + "Alternatively, a matching member initializer constructor might not be available for T " + "and should be defined."); }; std::apply(static_checker, drop_keys(std::tie(args...))); msgpack::type::define_map{ args... }.msgpack_pack(o); diff --git a/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp b/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp index 4d8d5476d1..1a37630188 100644 --- a/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp +++ b/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp @@ -9,26 +9,26 @@ struct GoodExample { barretenberg::fr a; barretenberg::fr b; - MSGPACK(a, b); + MSGPACK_FIELDS(a, b); } good_example; struct BadExampleOverlap { barretenberg::fr a; barretenberg::fr b; - MSGPACK(a, a); + MSGPACK_FIELDS(a, a); } bad_example_overlap; struct BadExampleIncomplete { barretenberg::fr a; barretenberg::fr b; - MSGPACK(a); + MSGPACK_FIELDS(a); } bad_example_incomplete; struct BadExampleCompileTimeError { std::vector a; barretenberg::fr b; - MSGPACK(b); // Type mismatch, expect 'a', will catch at compile-time + MSGPACK_FIELDS(b); // Type mismatch, expect 'a', will catch at compile-time } bad_example_compile_time_error; struct BadExampleOutOfObject { @@ -46,9 +46,9 @@ TEST(msgpack_tests, msgpack_sanity_sanity) { EXPECT_EQ(msgpack::check_msgpack_method(good_example), ""); EXPECT_EQ(msgpack::check_msgpack_method(bad_example_overlap), - "Overlap in BadExampleOverlap MSGPACK() params detected!"); + "Overlap in BadExampleOverlap MSGPACK_FIELDS() params detected!"); EXPECT_EQ(msgpack::check_msgpack_method(bad_example_incomplete), - "Incomplete BadExampleIncomplete MSGPACK() params! Not all of object specified."); + "Incomplete BadExampleIncomplete MSGPACK_FIELDS() params! Not all of object specified."); // If we actually try to msgpack BadExampleCompileTimeError we will statically error // This is great, but we need to check the underlying facility *somehow* @@ -62,7 +62,7 @@ TEST(msgpack_tests, msgpack_sanity_sanity) bad_example_compile_time_error.msgpack(checker); EXPECT_EQ(msgpack::check_msgpack_method(bad_example_out_of_object), - "Some BadExampleOutOfObject MSGPACK() params don't exist in object!"); + "Some BadExampleOutOfObject MSGPACK_FIELDS() params don't exist in object!"); } struct ComplicatedSchema { @@ -70,7 +70,7 @@ struct ComplicatedSchema { std::optional good_or_not; barretenberg::fr bare; std::variant huh; - MSGPACK(array, good_or_not, bare, huh); + MSGPACK_FIELDS(array, good_or_not, bare, huh); } complicated_schema; TEST(msgpack_tests, msgpack_schema_sanity) diff --git a/cpp/src/barretenberg/serialize/raw_pointer.hpp b/cpp/src/barretenberg/serialize/raw_pointer.hpp index 5b61b9ad1c..b10590be0c 100644 --- a/cpp/src/barretenberg/serialize/raw_pointer.hpp +++ b/cpp/src/barretenberg/serialize/raw_pointer.hpp @@ -1,25 +1,15 @@ #include #include "msgpack_impl/schema_name.hpp" -template -struct RawPointer { - T* prover {}; - void msgpack_pack(auto& packer) const { - packer.pack(reinterpret_cast(prover)); - } - void msgpack_unpack(auto object) { - prover = reinterpret_cast((uintptr_t)object); - } - T* operator->() { - return prover; - } +template struct RawPointer { + T* prover{}; + void msgpack_pack(auto& packer) const { packer.pack(reinterpret_cast(prover)); } + void msgpack_unpack(auto object) { prover = reinterpret_cast((uintptr_t)object); } + T* operator->() { return prover; } }; -namespace msgpack { // help our msgpack schema compiler with this struct -template -inline void schema_pack(auto& packer, RawPointer const&) +template inline void msgpack_schema_pack(auto& packer, RawPointer const&) { - packer.pack_alias(schema_name(T {}) + "Ptr", "int"); + packer.pack_alias(msgpack_schema_name(T{}) + "Ptr", "int"); } -} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/serialize/test_helper.hpp b/cpp/src/barretenberg/serialize/test_helper.hpp index eaed342245..903ea3c431 100644 --- a/cpp/src/barretenberg/serialize/test_helper.hpp +++ b/cpp/src/barretenberg/serialize/test_helper.hpp @@ -7,23 +7,32 @@ * @tparam T The object type. * @param object The object. Can be a default-initialized object. */ -template -std::pair msgpack_roundtrip(const T &object) { +template std::pair msgpack_roundtrip(const T& object) +{ T result; msgpack::sbuffer buffer; msgpack::pack(buffer, object); msgpack::unpack(buffer.data(), buffer.size()).get().convert(result); - return {object, result}; + return { object, result }; } -inline std::pair cbind_test_impl(auto cbind_func, auto... test_args) { - auto expected_ret = func(test_args...); - auto [input, input_len] = msgpack::encode_buffer(std::make_tuple(test_args...)); - uint8_t *output; +template inline T call_msgpack_cbind(auto cbind_func, auto... test_args) +{ + auto [input, input_len] = msgpack_encode_buffer(std::make_tuple(test_args...)); + uint8_t* output; size_t output_len; cbind_func(input, input_len, &output, &output_len); - decltype(expected_ret) actual_ret; - msgpack::decode(&actual_ret, output, output_len); + T actual_ret; + msgpack::unpack((const char*)output, output_len).get().convert(actual_ret); aligned_free(output); - return {msgpack::string_encode(actual_ret), msgpack::string_encode(expected_ret)}; + return actual_ret; +} + +// Running the end-to-end tests that msgpack bind creates +// This should suffice in testing the binding interface, function tests can be separate +inline auto call_func_and_wrapper(auto func, auto cbind_func, auto... test_args) +{ + auto expected_ret = func(test_args...); + auto actual_ret = call_msgpack_cbind(cbind_func, test_args...); + return std::make_pair(actual_ret, expected_ret); } diff --git a/cpp/src/barretenberg/stdlib/primitives/address/address.hpp b/cpp/src/barretenberg/stdlib/primitives/address/address.hpp index 9eccf401e7..aaed93c92d 100644 --- a/cpp/src/barretenberg/stdlib/primitives/address/address.hpp +++ b/cpp/src/barretenberg/stdlib/primitives/address/address.hpp @@ -145,10 +145,8 @@ template class address_t { } // namespace stdlib } // namespace proof_system::plonk -namespace msgpack { // help our msgpack schema compiler with this buffer alias (as far as wire representation is concerned) class -inline void schema_pack(auto& packer, proof_system::plonk::stdlib::address const&) +inline void msgpack_schema_pack(auto& packer, proof_system::plonk::stdlib::address const&) { packer.pack_alias("Address", "bin32"); } -} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp b/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp index f91dbf6040..1e97da3f77 100644 --- a/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp +++ b/cpp/src/barretenberg/stdlib/recursion/aggregation_state/native_aggregation_state.hpp @@ -21,7 +21,7 @@ struct native_aggregation_state { std::vector proof_witness_indices; bool has_data = false; - MSGPACK(P0, P1, public_inputs, proof_witness_indices, has_data); + MSGPACK_FIELDS(P0, P1, public_inputs, proof_witness_indices, has_data); bool operator==(native_aggregation_state const& other) const { return P0 == other.P0 && P1 == other.P1 && public_inputs == other.public_inputs && diff --git a/cpp/src/msgpack-c b/cpp/src/msgpack-c index d0448f5b76..1c90fb3a7c 160000 --- a/cpp/src/msgpack-c +++ b/cpp/src/msgpack-c @@ -1 +1 @@ -Subproject commit d0448f5b76dcf9f012be0d73300c3acf65c91adb +Subproject commit 1c90fb3a7c08466d67b15062713a02da2e073fef From f5c4f4b9543fad3f3263501b15f5b6b8b96fc843 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 18:00:00 -0400 Subject: [PATCH 72/92] chore: remove unused funcs --- cpp/src/barretenberg/serialize/cbind.hpp | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/cpp/src/barretenberg/serialize/cbind.hpp b/cpp/src/barretenberg/serialize/cbind.hpp index 9e889dda18..ed7a7c651b 100644 --- a/cpp/src/barretenberg/serialize/cbind.hpp +++ b/cpp/src/barretenberg/serialize/cbind.hpp @@ -35,28 +35,6 @@ inline std::pair msgpack_encode_buffer(auto&& obj) return { output, buffer.size() }; } -std::string msgpack_json_encode(const auto& obj) -{ - std::stringstream output; - msgpack::pack(output, obj); - std::string output_str = output.str(); - msgpack::object_handle oh = msgpack::unpack(output_str.data(), output_str.size()); - std::stringstream pretty_output; - pretty_output << oh.get() << std::endl; - return pretty_output.str(); -} - -void msgpack_json_print(const auto& obj) -{ - std::stringstream output; - msgpack::pack(output, obj); - std::string output_str = output.str(); - msgpack::object_handle oh = msgpack::unpack(output_str.data(), output_str.size()); - std::stringstream pretty_output; - pretty_output << oh.get() << std::endl; - logstr(pretty_output.str().c_str()); -} - template constexpr auto param_tuple() { return typename decltype(get_func_traits())::Args{}; From 4cc89e54c2dfcef4823ab9a39fe15c7edb0311cf Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 18:34:10 -0400 Subject: [PATCH 73/92] Revert fields constructor change for now --- cpp/src/barretenberg/ecc/curves/bn254/fr.hpp | 4 +--- cpp/src/barretenberg/ecc/fields/field.hpp | 10 +++------- .../serialize/msgpack_impl/schema_impl.hpp | 2 +- cpp/src/barretenberg/serialize/msgpack_schema.test.cpp | 4 ++-- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp b/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp index dcbfde4423..96f445e578 100644 --- a/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp +++ b/cpp/src/barretenberg/ecc/curves/bn254/fr.hpp @@ -67,10 +67,8 @@ typedef field fr; } // namespace barretenberg -namespace msgpack { // define this as a named alias in msgpack schema generation -inline void schema_pack(auto& packer, barretenberg::fr const&) +inline void msgpack_schema_pack(auto& packer, barretenberg::fr const&) { packer.pack_alias("Fr", "bin32"); } -} // namespace msgpack \ No newline at end of file diff --git a/cpp/src/barretenberg/ecc/fields/field.hpp b/cpp/src/barretenberg/ecc/fields/field.hpp index 4bd9a94d9b..05d201c479 100644 --- a/cpp/src/barretenberg/ecc/fields/field.hpp +++ b/cpp/src/barretenberg/ecc/fields/field.hpp @@ -24,16 +24,12 @@ namespace barretenberg { template struct alignas(32) field { public: - // We don't initialize data in the default constructor since we'd lose a lot of time on huge array initializations. + // We don't initialize data by default since we'd lose a lot of time on pointless initializations. // Other alternatives have been noted, such as casting to get around constructors where they matter, // however it is felt that sanitizer tools (e.g. MSAN) can detect garbage well, whereas doing // hacky casts where needed would require rework to critical algos like MSM, FFT, Sumcheck. - // Instead, the recommended solution is use an explicit {} where initialization is important: - // field f; // not initialized - // field f{}; // zero-initialized - // std::array arr; // not initialized, good for huge N - // std::array arr {}; // zero-initialized, preferable for moderate N - field() = default; + // Instead, the recommended solution is use an explicit = 0 where initialization is important. + field() noexcept {} constexpr field(const uint256_t& input) noexcept : data{ input.data[0], input.data[1], input.data[2], input.data[3] } diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp index e67e68edee..67c2fdea03 100644 --- a/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp +++ b/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp @@ -171,7 +171,7 @@ inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::array co * @param obj The object to print schema of. * @return std::string The schema as a string. */ -std::string msgpack_schema_to_string(auto obj) +inline std::string msgpack_schema_to_string(auto obj) { msgpack::sbuffer output; MsgpackSchemaPacker printer{ output }; diff --git a/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp b/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp index 1a37630188..516caac5f5 100644 --- a/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp +++ b/cpp/src/barretenberg/serialize/msgpack_schema.test.cpp @@ -76,9 +76,9 @@ struct ComplicatedSchema { TEST(msgpack_tests, msgpack_schema_sanity) { EXPECT_EQ( - msgpack::schema_to_string(good_example), + msgpack_schema_to_string(good_example), "{\"__typename\":\"GoodExample\",\"a\":[\"alias\",[\"Fr\",\"bin32\"]],\"b\":[\"alias\",[\"Fr\",\"bin32\"]]}\n"); - EXPECT_EQ(msgpack::schema_to_string(complicated_schema), + EXPECT_EQ(msgpack_schema_to_string(complicated_schema), "{\"__typename\":\"ComplicatedSchema\",\"array\":[\"vector\",[[\"array\",[[\"alias\",[\"Fr\",\"bin32\"]]," "20]]]],\"good_or_not\":[\"optional\",[{\"__typename\":\"GoodExample\",\"a\":[\"alias\",[\"Fr\"," "\"bin32\"]],\"b\":[\"alias\",[\"Fr\",\"bin32\"]]}]],\"bare\":[\"alias\",[\"Fr\",\"bin32\"]],\"huh\":[" From 264225c30d663b7d9b14f7237c44526be020fcac Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 18:44:05 -0400 Subject: [PATCH 74/92] chore: Revert .circleci changes --- .circleci/build | 134 ---------------------------- .circleci/cond_spot_run_build | 26 ------ .circleci/cond_spot_run_test_script | 2 - .circleci/init_submodules | 18 ---- .circleci/remote_build/remote_build | 28 ------ 5 files changed, 208 deletions(-) delete mode 100755 .circleci/build delete mode 100755 .circleci/cond_spot_run_build delete mode 100755 .circleci/init_submodules delete mode 100755 .circleci/remote_build/remote_build diff --git a/.circleci/build b/.circleci/build deleted file mode 100755 index 83c573dadd..0000000000 --- a/.circleci/build +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -# -# Builds a docker image and pushes it to it's repository. Leverages caches where possible. -# Cached images include previous successfully built images built on this branch. -# The images output are cache images, meaning they will eventually get purged. -# The deploy phase will tag the images such that they become permanent. -# -# usage: ./build -# example: ./build barretenberg-x86_64-linux-clang -# output image: -# 278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-x86_64-linux-clang:cache-deadbeefcafebabe1337c0d3 - -set -e - -REPOSITORY=$1 -DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) -PROJECT_DIR=$(query_manifest projectDir $REPOSITORY) - -echo "Repository: $REPOSITORY" -echo "Working directory: $PWD" -echo "Dockerfile: $DOCKERFILE" - -init_submodules $REPOSITORY - -cd $(query_manifest buildDir $REPOSITORY) - -function fetch_image() { - echo "Pulling: $1" - if ! docker pull $1 > /dev/null 2>&1; then - echo "Image not found: $1" - return 1 - fi - return 0 -} - -# Ensure ECR repository exists. -ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle - -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY) -echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT" - -# If we have previously successful commit, we can early out if nothing relevant has changed since. -if ! check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then - echo "No rebuild necessary. Retagging..." - STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') - for STAGE in $STAGES; do - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true - done - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH - untag_remote_image $REPOSITORY tainted - exit 0 -fi - -# Validate any terraform if it exists. -if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then - ensure_terraform - export TF_IN_AUTOMATION=1 - pushd $ROOT_PATH/$PROJECT_DIR/terraform - for DIR in . $(find . -maxdepth 1 -type d); do - pushd $DIR - if [ -f ./main.tf ]; then - terraform init -input=false -backend-config="key=dummy" - terraform validate - fi - popd - done - popd -fi - -# Pull latest parents that are not ours. -echo "$DOCKERHUB_PASSWORD" | docker login -u aztecprotocolci --password-stdin -PARENTS=$(cat $DOCKERFILE | sed -n -e 's/^FROM \([^[:space:]]\+\).*/\1/p' | grep -v $ECR_DEPLOY_URL | sort | uniq) -for PARENT in $PARENTS; do - fetch_image $PARENT -done - -# For each parent that's ours, pull in the latest image. -PARENTS=$(cat $DOCKERFILE | sed -n -e "s/^FROM $ECR_DEPLOY_URL\/\([^[:space:]]\+\).*/\1/p") -for PARENT in $PARENTS; do - # Extract repository name (i.e. discard tag). - PARENT_REPO=${PARENT%:*} - PARENT_COMMIT_HASH=$(last_successful_commit $PARENT_REPO) - # There must be a parent image to continue. - if [ -z "$PARENT_COMMIT_HASH" ]; then - echo "No parent image found for $PARENT_REPO" - exit 1 - fi - PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:cache-$PARENT_COMMIT_HASH - echo "Pulling dependency $PARENT_REPO..." - fetch_image $PARENT_IMAGE_URI - # Tag it to look like an official release as that's what we use in Dockerfiles. - docker tag $PARENT_IMAGE_URI $ECR_DEPLOY_URL/$PARENT -done - -# Pull, build and push each named stage to cache. -STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') -for STAGE in $STAGES; do - # Get the last build of this stage to leverage layer caching. - if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then - echo "Pulling stage: $STAGE" - STAGE_IMAGE_LAST_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT-$STAGE - if fetch_image $STAGE_IMAGE_LAST_URI; then - STAGE_CACHE_FROM="--cache-from $STAGE_IMAGE_LAST_URI" - fi - fi - - echo "Building stage: $STAGE" - STAGE_IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH-$STAGE - docker build --target $STAGE $STAGE_CACHE_FROM -t $STAGE_IMAGE_COMMIT_URI -f $DOCKERFILE . - - # We don't want to have redo this stages work when building the final image. Use it as a layer cache. - CACHE_FROM="--cache-from $STAGE_IMAGE_COMMIT_URI $CACHE_FROM" - - echo "Pushing stage: $STAGE" - docker push $STAGE_IMAGE_COMMIT_URI > /dev/null 2>&1 - echo -done - -# Pull previous image to use it as a layer cache if it exists. -if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then - LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT - echo "Pulling previous build of $REPOSITORY..." - fetch_image $LAST_SUCCESSFUL_URI || true - CACHE_FROM="--cache-from $LAST_SUCCESSFUL_URI $CACHE_FROM" - echo -fi - -# Build the actual image and give it a commit tag. -IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH -echo "Building image: $IMAGE_COMMIT_URI" -docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG . -echo "Pushing image: $IMAGE_COMMIT_URI" -docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 -untag_remote_image $REPOSITORY tainted \ No newline at end of file diff --git a/.circleci/cond_spot_run_build b/.circleci/cond_spot_run_build deleted file mode 100755 index 21fef601a6..0000000000 --- a/.circleci/cond_spot_run_build +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -e - -REPOSITORY=$1 -shift -SPEC=$1 -shift -DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) - -init_submodules $REPOSITORY - -cd $(query_manifest buildDir $REPOSITORY) - -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY) -echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT" - -if check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then - spot_run_script $SPEC ../.circleci/remote_build/remote_build $REPOSITORY $@ -else - echo "No rebuild necessary. Retagging..." - STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') - for STAGE in $STAGES; do - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true - done - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH -fi \ No newline at end of file diff --git a/.circleci/cond_spot_run_test_script b/.circleci/cond_spot_run_test_script index 66d8cc5c27..044777b673 100755 --- a/.circleci/cond_spot_run_test_script +++ b/.circleci/cond_spot_run_test_script @@ -5,8 +5,6 @@ SCRIPT_PATH=$2 shift shift -init_submodules $REPOSITORY - cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs diff --git a/.circleci/init_submodules b/.circleci/init_submodules deleted file mode 100755 index 939c8e4730..0000000000 --- a/.circleci/init_submodules +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# For a given repository, init any required submodules. -set -euo pipefail - -REPOSITORY=$1 - -BUILD_DIR=$(query_manifest buildDir $REPOSITORY) -if [ -d "$BUILD_DIR" ]; then - echo "Initialising any submodules under $REPOSITORY buildDir: $BUILD_DIR" - git submodule update --init --recursive $BUILD_DIR && exit || echo "No submodules under buildDir" -fi - -SUBMODULE_PATH=$(query_manifest submodulePath $REPOSITORY) -if [ -n "$SUBMODULE_PATH" ]; then - # TODO: Needs to actually init all dependency submodules as well. - echo "Initialising submodule: $SUBMODULE_PATH" - git submodule update --init --recursive $SUBMODULE_PATH -fi \ No newline at end of file diff --git a/.circleci/remote_build/remote_build b/.circleci/remote_build/remote_build deleted file mode 100755 index f8c7b3c806..0000000000 --- a/.circleci/remote_build/remote_build +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -e - -ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - -echo "Initialising remote build..." - -# Shallow checkout this commit. -mkdir project -cd project -git init -git remote add origin $GIT_REPOSITORY_URL -# Configure to only download metadata when fetching. -git config remote.origin.promisor true -git config remote.origin.partialclonefilter blob:none -git fetch --depth 50 origin $COMMIT_HASH -git checkout FETCH_HEAD -git submodule update --init - -echo "Git operations completed..." - -cd .circleci -BASH_ENV=/tmp/bash_env -echo "Calling setup env..." -source ./setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" -cd .. -echo "Calling build..." -build $@ From 0e0df3c27d5e00909545cffae3995d9b463c04ec Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 18:45:02 -0400 Subject: [PATCH 75/92] chore: Revert foundation removal --- .gitmodules | 6 +++--- foundation | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 160000 foundation diff --git a/.gitmodules b/.gitmodules index 0a2414020a..ae63c99f6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ -[submodule "cpp/src/msgpack-c"] - path = cpp/src/msgpack-c - url = https://github.com/AztecProtocol/msgpack-c +[submodule "foundation"] + path = foundation + url = git@github.com:AztecProtocol/foundation.git [submodule "sol/lib/forge-std"] path = sol/lib/forge-std url = https://github.com/foundry-rs/forge-std diff --git a/foundation b/foundation new file mode 160000 index 0000000000..003b3a640d --- /dev/null +++ b/foundation @@ -0,0 +1 @@ +Subproject commit 003b3a640d9b98a002d6c231ee75cdf0136e8116 From e5d8fe5dbd303b11fc4bbc7cd8859b4ef8f60f92 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 18:51:26 -0400 Subject: [PATCH 76/92] Revert .gitmodules --- .gitmodules | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitmodules b/.gitmodules index ae63c99f6f..7f651c8fde 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ +[submodule "cpp/src/msgpack-c"] + path = cpp/src/msgpack-c + url = https://github.com/AztecProtocol/msgpack-c [submodule "foundation"] path = foundation url = git@github.com:AztecProtocol/foundation.git From 88f79c724f1ded470456a0f6576a61a074ba660b Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 18:53:42 -0400 Subject: [PATCH 77/92] Update affine_element.hpp --- cpp/src/barretenberg/ecc/groups/affine_element.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/ecc/groups/affine_element.hpp b/cpp/src/barretenberg/ecc/groups/affine_element.hpp index f41787661c..312b01bfa3 100644 --- a/cpp/src/barretenberg/ecc/groups/affine_element.hpp +++ b/cpp/src/barretenberg/ecc/groups/affine_element.hpp @@ -177,7 +177,7 @@ template class alignas(64) affine_el } Fq x; Fq y; - // for serialization: update up with new fields + // for serialization: update with new fields MSGPACK_FIELDS(x, y); }; From a50c3389838dca998bf8ad0722851bac5c6f648b Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 18:55:35 -0400 Subject: [PATCH 78/92] Update element.hpp --- cpp/src/barretenberg/ecc/groups/element.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/ecc/groups/element.hpp b/cpp/src/barretenberg/ecc/groups/element.hpp index 430db6abf7..3f93f0f552 100644 --- a/cpp/src/barretenberg/ecc/groups/element.hpp +++ b/cpp/src/barretenberg/ecc/groups/element.hpp @@ -123,7 +123,7 @@ template class alignas(32) element { // } // return { x, y, Fq::one() }; // } - // for serialization: update up with new fields + // for serialization: update with new fields MSGPACK_FIELDS(x, y, z); static void conditional_negate_affine(const affine_element& in, From cb01cf9f624e42e668fe1acceb3312873e131968 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 18:57:22 -0400 Subject: [PATCH 79/92] Revert header optimizations --- .../composer_helper/standard_plonk_composer_helper.cpp | 2 -- .../composer_helper/turbo_plonk_composer_helper.cpp | 2 -- .../composer_helper/ultra_plonk_composer_helper.cpp | 2 -- cpp/src/barretenberg/plonk/composer/turbo_composer.cpp | 3 --- cpp/src/barretenberg/plonk/composer/ultra_composer.cpp | 3 --- .../plonk/proof_system/verification_key/verification_key.hpp | 2 +- cpp/src/barretenberg/solidity_helpers/proof_gen.cpp | 4 ---- .../barretenberg/stdlib/recursion/verifier/verifier.test.cpp | 3 --- 8 files changed, 1 insertion(+), 20 deletions(-) diff --git a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/standard_plonk_composer_helper.cpp b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/standard_plonk_composer_helper.cpp index b9c350658a..dabf5e4bc1 100644 --- a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/standard_plonk_composer_helper.cpp +++ b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/standard_plonk_composer_helper.cpp @@ -8,8 +8,6 @@ #include "barretenberg/proof_system/composer/composer_helper_lib.hpp" #include "barretenberg/plonk/composer/splitting_tmp/composer_helper/composer_helper_lib.hpp" #include "barretenberg/proof_system/composer/permutation_helper.hpp" -// Be explicit about including the heavy implementation headers: -#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" #include #include diff --git a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/turbo_plonk_composer_helper.cpp b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/turbo_plonk_composer_helper.cpp index 7a335291df..c367d9ccd1 100644 --- a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/turbo_plonk_composer_helper.cpp +++ b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/turbo_plonk_composer_helper.cpp @@ -13,8 +13,6 @@ #include "barretenberg/proof_system/composer/permutation_helper.hpp" #include "barretenberg/proof_system/composer/composer_helper_lib.hpp" #include "barretenberg/plonk/composer/splitting_tmp/composer_helper/composer_helper_lib.hpp" -// Be explicit about including the heavy implementation headers: -#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" using namespace barretenberg; diff --git a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/ultra_plonk_composer_helper.cpp b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/ultra_plonk_composer_helper.cpp index f5b1714c5b..6dd56226e1 100644 --- a/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/ultra_plonk_composer_helper.cpp +++ b/cpp/src/barretenberg/plonk/composer/splitting_tmp/composer_helper/ultra_plonk_composer_helper.cpp @@ -5,8 +5,6 @@ #include "barretenberg/proof_system/circuit_constructors/ultra_circuit_constructor.hpp" #include "barretenberg/proof_system/composer/permutation_helper.hpp" #include "barretenberg/plonk/proof_system/commitment_scheme/kate_commitment_scheme.hpp" -// Be explicit about including the heavy implementation headers: -#include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp" #include #include diff --git a/cpp/src/barretenberg/plonk/composer/turbo_composer.cpp b/cpp/src/barretenberg/plonk/composer/turbo_composer.cpp index bee4a8a90b..f3339cf147 100644 --- a/cpp/src/barretenberg/plonk/composer/turbo_composer.cpp +++ b/cpp/src/barretenberg/plonk/composer/turbo_composer.cpp @@ -2,9 +2,6 @@ #include "barretenberg/ecc/curves/bn254/scalar_multiplication/scalar_multiplication.hpp" #include "barretenberg/numeric/bitop/get_msb.hpp" #include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp" -// Be explicit about including the heavy implementation headers: -#include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp" -#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" #include "barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_arithmetic_widget.hpp" #include "barretenberg/plonk/proof_system/widgets/transition_widgets/fixed_base_widget.hpp" #include "barretenberg/plonk/proof_system/widgets/transition_widgets/turbo_logic_widget.hpp" diff --git a/cpp/src/barretenberg/plonk/composer/ultra_composer.cpp b/cpp/src/barretenberg/plonk/composer/ultra_composer.cpp index 1eb5a4fcc6..1f3e9fb3ed 100644 --- a/cpp/src/barretenberg/plonk/composer/ultra_composer.cpp +++ b/cpp/src/barretenberg/plonk/composer/ultra_composer.cpp @@ -11,9 +11,6 @@ #include "barretenberg/plonk/proof_system/widgets/transition_widgets/plookup_auxiliary_widget.hpp" #include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp" #include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget.hpp" -// Be explicit about including the heavy implementation headers: -#include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp" -#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" #include "barretenberg/plonk/proof_system/commitment_scheme/kate_commitment_scheme.hpp" #include "barretenberg/srs/reference_string/file_reference_string.hpp" diff --git a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp index fb1aceb828..0ee17d60e6 100644 --- a/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/verification_key/verification_key.hpp @@ -94,7 +94,7 @@ struct verification_key { std::vector recursive_proof_public_input_indices; size_t program_width = 3; - // for serialization: update up with new fields + // for serialization: update with new fields void msgpack_pack(auto& packer) const { verification_key_data data = { composer_type, diff --git a/cpp/src/barretenberg/solidity_helpers/proof_gen.cpp b/cpp/src/barretenberg/solidity_helpers/proof_gen.cpp index 4c0368370b..4765895ec5 100644 --- a/cpp/src/barretenberg/solidity_helpers/proof_gen.cpp +++ b/cpp/src/barretenberg/solidity_helpers/proof_gen.cpp @@ -5,10 +5,6 @@ #include "barretenberg/plonk/composer/standard_composer.hpp" #include "barretenberg/plonk/composer/ultra_composer.hpp" -// Be explicit about including the heavy implementation headers: -#include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp" -#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" - #include "circuits/blake_circuit.hpp" #include "circuits/add_2_circuit.hpp" #include "circuits/recursive_circuit.hpp" diff --git a/cpp/src/barretenberg/stdlib/recursion/verifier/verifier.test.cpp b/cpp/src/barretenberg/stdlib/recursion/verifier/verifier.test.cpp index 615112e262..82e8042b02 100644 --- a/cpp/src/barretenberg/stdlib/recursion/verifier/verifier.test.cpp +++ b/cpp/src/barretenberg/stdlib/recursion/verifier/verifier.test.cpp @@ -3,9 +3,6 @@ #include "barretenberg/transcript/transcript.hpp" #include "barretenberg/plonk/proof_system/proving_key/serialize.hpp" #include "barretenberg/stdlib/primitives/curves/bn254.hpp" -// Be explicit about including the heavy implementation headers: -#include "barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget_impl.hpp" -#include "barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget_impl.hpp" #include "barretenberg/ecc/curves/bn254/fq12.hpp" #include "barretenberg/ecc/curves/bn254/pairing.hpp" #include "../../hash/blake3s/blake3s.hpp" From 17e61fa52d1b2da413ab7e9a44df589ed3ef3412 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 18:57:59 -0400 Subject: [PATCH 80/92] Revert init line --- cpp/src/barretenberg/stdlib/commitment/pedersen/pedersen.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/cpp/src/barretenberg/stdlib/commitment/pedersen/pedersen.cpp b/cpp/src/barretenberg/stdlib/commitment/pedersen/pedersen.cpp index 6288b92314..c6212093c2 100644 --- a/cpp/src/barretenberg/stdlib/commitment/pedersen/pedersen.cpp +++ b/cpp/src/barretenberg/stdlib/commitment/pedersen/pedersen.cpp @@ -79,9 +79,6 @@ field_t pedersen_commitment::compress_unsafe(const field_t& in_left, return pedersen_plookup_commitment::compress({ in_left, in_right }); } - // Make sure generators exist for pedersen (will do nothing if already initialized) - crypto::generators::init_generator_data(); - std::vector accumulators; generator_index_t index_1 = { hash_index, 0 }; generator_index_t index_2 = { hash_index, 1 }; From 47b4ea931d87e0add421007286b99e525ed5a09b Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 15 May 2023 19:00:38 -0400 Subject: [PATCH 81/92] Update polynomial_store.test.cpp --- .../proof_system/polynomial_store/polynomial_store.test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp index a781ca4442..a068cce790 100644 --- a/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp +++ b/cpp/src/barretenberg/proof_system/polynomial_store/polynomial_store.test.cpp @@ -102,7 +102,7 @@ TEST(PolynomialStore, RangeBasedFor) polynomial_store.put("id_2", std::move(poly2)); // Check that PolynomialStore meets criteria for std::ranges::range - // EXPECT_TRUE(std::ranges::range>); + EXPECT_TRUE(std::ranges::range>); // For example ... Polynomial polynomial_sum(size); From 33aceca169c56d14442123dbacab0f7586cef949 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 16 May 2023 09:29:34 -0400 Subject: [PATCH 82/92] Revert header optimization --- .../proof_system/widgets/random_widgets/permutation_widget.hpp | 2 -- .../proof_system/widgets/random_widgets/plookup_widget.hpp | 2 -- 2 files changed, 4 deletions(-) diff --git a/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp b/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp index 6ae47ded0a..98e58b0bad 100644 --- a/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/permutation_widget.hpp @@ -44,6 +44,4 @@ class ProverPermutationWidget : public ProverRandomWidget { } // namespace proof_system::plonk -// The class implementation is expensive to compile -// Include it only in translation (.cpp) units that need it #include "./permutation_widget_impl.hpp" \ No newline at end of file diff --git a/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget.hpp b/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget.hpp index b76697e415..623f3a6193 100644 --- a/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget.hpp +++ b/cpp/src/barretenberg/plonk/proof_system/widgets/random_widgets/plookup_widget.hpp @@ -48,6 +48,4 @@ class ProverPlookupWidget : public ProverRandomWidget { } // namespace proof_system::plonk -// The class implementation is expensive to compile -// Include it only in translation (.cpp) units that need it #include "./plookup_widget_impl.hpp" \ No newline at end of file From af1e97e7bf91174b480dc926c01155afda1d8bba Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 16 May 2023 10:26:41 -0400 Subject: [PATCH 83/92] Update raw_pointer.hpp --- .../barretenberg/serialize/raw_pointer.hpp | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/cpp/src/barretenberg/serialize/raw_pointer.hpp b/cpp/src/barretenberg/serialize/raw_pointer.hpp index b10590be0c..9d058a426a 100644 --- a/cpp/src/barretenberg/serialize/raw_pointer.hpp +++ b/cpp/src/barretenberg/serialize/raw_pointer.hpp @@ -1,11 +1,32 @@ #include #include "msgpack_impl/schema_name.hpp" -template struct RawPointer { - T* prover{}; - void msgpack_pack(auto& packer) const { packer.pack(reinterpret_cast(prover)); } - void msgpack_unpack(auto object) { prover = reinterpret_cast((uintptr_t)object); } - T* operator->() { return prover; } +// Holds a raw pointer to an object of type T. +// It provides methods for packing and unpacking the pointer using MessagePack, +// a binary serialization format. +template +struct RawPointer { + // Raw pointer to an object of type T + T* ptr = nullptr; + + // Pack the raw pointer into a MessagePack packer. + // The pointer is first cast to an integer type (uintptr_t) which can hold a pointer, + // and then packed into the packer. + void msgpack_pack(auto& packer) const { + packer.pack(reinterpret_cast(ptr)); + } + + // Unpack the raw pointer from a MessagePack object. + // The object is first cast to an integer type (uintptr_t), and then to a pointer of type T. + void msgpack_unpack(auto object) { + ptr = reinterpret_cast((uintptr_t)object); + } + + // Overload the arrow operator to return the raw pointer. + // This allows users to directly access the object pointed to by the raw pointer. + T* operator->() { + return ptr; + } }; // help our msgpack schema compiler with this struct From e3c8f9d1a57ecc8893b1e0bff434742780edb6f1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 16 May 2023 10:28:46 -0400 Subject: [PATCH 84/92] Update raw_pointer.hpp --- cpp/src/barretenberg/serialize/raw_pointer.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/barretenberg/serialize/raw_pointer.hpp b/cpp/src/barretenberg/serialize/raw_pointer.hpp index 9d058a426a..4b6e0b16b0 100644 --- a/cpp/src/barretenberg/serialize/raw_pointer.hpp +++ b/cpp/src/barretenberg/serialize/raw_pointer.hpp @@ -1,3 +1,4 @@ +#pragma once #include #include "msgpack_impl/schema_name.hpp" From 712ca1c66ba6ce9b6cebfdc5f4e2dd24b21b35e6 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 16 May 2023 13:24:30 -0400 Subject: [PATCH 85/92] Update func_traits.hpp documentation --- .../serialize/msgpack_impl/func_traits.hpp | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/func_traits.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/func_traits.hpp index 5805bc4cea..a514ebbba1 100644 --- a/cpp/src/barretenberg/serialize/msgpack_impl/func_traits.hpp +++ b/cpp/src/barretenberg/serialize/msgpack_impl/func_traits.hpp @@ -2,39 +2,57 @@ #include #include "../msgpack.hpp" -template struct func_traits; +// Define a template struct to deduce function traits for different function types +template +struct func_traits; -template struct func_traits { - typedef std::tuple Args; - Args args; - R ret; - MSGPACK_FIELDS(args, ret); +// Specialization for function pointers +template +struct func_traits { + typedef std::tuple Args; // Define a tuple type that holds all argument types + Args args; // Args instance + R ret; // Holds return type + MSGPACK_FIELDS(args, ret); // Macro from msgpack library to serialize/deserialize fields }; -template struct func_traits { + +// Specialization for function references +template +struct func_traits { typedef std::tuple Args; Args args; R ret; MSGPACK_FIELDS(args, ret); }; -template struct func_traits { +// Specialization for member function pointers. This also includes lambda types, +// as they are functors (objects with operator()) and hence have a member function pointer +template +struct func_traits { typedef std::tuple Args; Args args; R ret; MSGPACK_FIELDS(args, ret); }; -// Template metamagic for figuring out the parameters and return type of a function +// Define a concept that checks if the type is a lambda (or functor) type +// This is done by checking if T::operator() exists template -concept Callable = - requires() { typename std::enable_if_t, void>; }; +concept LambdaType = requires() { + typename std::enable_if_t, void>; +}; -template constexpr auto get_func_traits() +// Overload for lambda (or functor) types +template +constexpr auto get_func_traits() { + // If T is a lambda type (i.e. it has operator()), deduce its traits using func_traits return func_traits(); } -template constexpr auto get_func_traits() +// Overload for non-lambda types +template +constexpr auto get_func_traits() { + // If T is not a lambda, just deduce its traits using func_traits return func_traits(); } From d5ae7b4e081f26e529f695bf190f9c8bcee095b0 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 16 May 2023 22:41:03 -0400 Subject: [PATCH 86/92] Document msgpack methods in field_impl.hpp --- .../barretenberg/ecc/fields/field_impl.hpp | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/cpp/src/barretenberg/ecc/fields/field_impl.hpp b/cpp/src/barretenberg/ecc/fields/field_impl.hpp index 87dea05856..a486aab45f 100644 --- a/cpp/src/barretenberg/ecc/fields/field_impl.hpp +++ b/cpp/src/barretenberg/ecc/fields/field_impl.hpp @@ -606,28 +606,44 @@ template constexpr field field::multiplicative_generator() noexc return target; } -// For serialization +// This function is used to serialize a field. It matches the old serialization format by first +// converting the field from Montgomery form, which is a special representation used for efficient +// modular arithmetic. template void field::msgpack_pack(auto& packer) const { + // The field is first converted from Montgomery form, similar to how the old format did it. auto adjusted = from_montgomery_form(); + + // The data is then converted to big endian format using htonll, which stands for "host to network long long". + // This is necessary because the data will be written to a raw msgpack buffer, which requires big endian format. uint64_t bin_data[4] = { htonll(adjusted.data[3]), htonll(adjusted.data[2]), htonll(adjusted.data[1]), htonll(adjusted.data[0]) }; + + // The packer is then used to write the binary data to the buffer, just like in the old format. packer.pack_bin(sizeof(bin_data)); packer.pack_bin_body((const char*)bin_data, sizeof(bin_data)); } -// For deserialization +// This function is used to deserialize a field. It also matches the old deserialization format by +// reading the binary data as big endian uint64_t's, correcting them to the host endianness, and +// then converting the field back to Montgomery form. template void field::msgpack_unpack(auto o) { - // Trigger msgpack specialization for binary + // The binary data is first extracted from the msgpack object. std::array raw_data = o; - // With the binary data, we have to read it as big endian uint64_t's, then correct them to the host endianness + + // The binary data is then read as big endian uint64_t's. This is done by casting the raw data to uint64_t* and then + // using ntohll ("network to host long long") to correct the endianness to the host's endianness. uint64_t* cast_data = (uint64_t*)&raw_data[0]; uint64_t reversed[] = {ntohll(cast_data[3]), ntohll(cast_data[2]), ntohll(cast_data[1]), ntohll(cast_data[0])}; + + // The corrected data is then copied back into the field's data array. for (int i = 0; i < 4; i++) { data[i] = reversed[i]; } + + // Finally, the field is converted back to Montgomery form, just like in the old format. *this = to_montgomery_form(); } From e3d0368a9510c1852a1f93484c3fbac28c40420a Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 16 May 2023 22:43:23 -0400 Subject: [PATCH 87/92] Update msgpack.hpp --- cpp/src/barretenberg/serialize/msgpack.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/serialize/msgpack.hpp b/cpp/src/barretenberg/serialize/msgpack.hpp index 6d418ba766..9f83eccd6a 100644 --- a/cpp/src/barretenberg/serialize/msgpack.hpp +++ b/cpp/src/barretenberg/serialize/msgpack.hpp @@ -1,5 +1,5 @@ #pragma once -/* Minimal header for msgpack. This should be included, *not* msgpack.hpp directly. +/* Minimal header for msgpack. This should be included as "barretenberg/serialize/msgpack.hpp", *not* msgpack.hpp directly. That should be treated as an implementation-only header. ## Overview From 6c1696495ac1eb6f7c0e55f7aecdef95450e286b Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 16 May 2023 22:49:27 -0400 Subject: [PATCH 88/92] Update cbind.hpp --- cpp/src/barretenberg/serialize/cbind.hpp | 26 +++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/cpp/src/barretenberg/serialize/cbind.hpp b/cpp/src/barretenberg/serialize/cbind.hpp index ed7a7c651b..a047615270 100644 --- a/cpp/src/barretenberg/serialize/cbind.hpp +++ b/cpp/src/barretenberg/serialize/cbind.hpp @@ -35,17 +35,41 @@ inline std::pair msgpack_encode_buffer(auto&& obj) return { output, buffer.size() }; } +// This is a template function that will return the argument types +// of a given function type T as a tuple. template constexpr auto param_tuple() { + // decltype is used to determine the type of an expression at compile-time. + // get_func_traits() is assumed to return a structure whose ::Args member is a tuple + // of argument types of function T. This function constructs an instance of that tuple and returns it. return typename decltype(get_func_traits())::Args{}; } +// This function is intended to bind a function to a MessagePack-formatted input data, +// perform the function with the unpacked data, then pack the result back into MessagePack format. inline void msgpack_cbind_impl( - auto func, const uint8_t* input_in, size_t input_len_in, uint8_t** output_out, size_t* output_len_out) + auto func, // The function to be applied + const uint8_t* input_in, // The input data in MessagePack format + size_t input_len_in, // The length of the input data + uint8_t** output_out, // The output data in MessagePack format + size_t* output_len_out) // The length of the output data { + // Get the parameter types of the function as a tuple. auto params = param_tuple(); + + // Unpack the input data into the parameter tuple. + // msgpack::unpack takes a buffer and its size, and returns an object_handle. + // Calling .get() on that handle yields an object, and calling .convert on that + // object converts it into the given type, in this case, the parameter tuple for func. msgpack::unpack((const char*)input_in, input_len_in).get().convert(params); + + // Apply the function to the parameters, then encode the result into a MessagePack buffer. + // std::apply takes a function and a tuple, and applies the function to the tuple's elements. + // msgpack_encode_buffer is assumed to take the result of the function and return a pair + // consisting of a pointer to the output buffer and its size. auto [output, output_len] = msgpack_encode_buffer(std::apply(func, params)); + + // Assign the output data and its length to the given output parameters. *output_out = output; *output_len_out = output_len; } From 4e7f48c3702def56a2f9ce7ac6be7ba5c0a6a179 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 16 May 2023 22:51:10 -0400 Subject: [PATCH 89/92] Update msgpack.hpp --- cpp/src/barretenberg/serialize/msgpack.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/src/barretenberg/serialize/msgpack.hpp b/cpp/src/barretenberg/serialize/msgpack.hpp index 9f83eccd6a..de8173b4fc 100644 --- a/cpp/src/barretenberg/serialize/msgpack.hpp +++ b/cpp/src/barretenberg/serialize/msgpack.hpp @@ -1,6 +1,7 @@ #pragma once -/* Minimal header for msgpack. This should be included as "barretenberg/serialize/msgpack.hpp", *not* msgpack.hpp directly. -That should be treated as an implementation-only header. +/* Minimal header for declaring msgpack fields. +This should be included as "barretenberg/serialize/msgpack.hpp", *not* msgpack.hpp directly +unless one plans to use msgpack in their translation unit. ## Overview From 4533702537409452d8089ab1ab7d0773af13efb4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 16 May 2023 22:55:26 -0400 Subject: [PATCH 90/92] Update msgpack.hpp --- cpp/src/barretenberg/serialize/msgpack.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/src/barretenberg/serialize/msgpack.hpp b/cpp/src/barretenberg/serialize/msgpack.hpp index de8173b4fc..767a2401e2 100644 --- a/cpp/src/barretenberg/serialize/msgpack.hpp +++ b/cpp/src/barretenberg/serialize/msgpack.hpp @@ -1,7 +1,7 @@ #pragma once /* Minimal header for declaring msgpack fields. -This should be included as "barretenberg/serialize/msgpack.hpp", *not* msgpack.hpp directly -unless one plans to use msgpack in their translation unit. +This should be included as "barretenberg/serialize/msgpack.hpp" unless a translation wants +to use msgpack for bindings, then "barretenberg/serialize/cbind.hpp" should be included. ## Overview @@ -96,6 +96,8 @@ e.g. unpacking #include "msgpack_impl/concepts.hpp" // Helper for above documented syntax +// Define a macro that takes any amount of parameters and expands to a msgpack method definition +// __VA__ARGS__ expands to the parmeters, comma separated. #define MSGPACK_FIELDS(...) \ void msgpack(auto pack_fn) \ { \ From 3ff51bb70c1b045888b0712d211823de3dad0f9e Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 16 May 2023 23:00:40 -0400 Subject: [PATCH 91/92] Update schema_impl.hpp --- .../barretenberg/serialize/msgpack_impl/schema_impl.hpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp b/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp index 67c2fdea03..a99038a1b3 100644 --- a/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp +++ b/cpp/src/barretenberg/serialize/msgpack_impl/schema_impl.hpp @@ -34,8 +34,10 @@ struct MsgpackSchemaPacker : msgpack::packer { */ void pack_alias(const std::string& schema_name, const std::string& msgpack_name) { + // We will pack a size 2 tuple pack_array(2); pack("alias"); + // That has a size 2 tuple as its 2nd arg pack_array(2); pack(schema_name); pack(msgpack_name); @@ -102,9 +104,11 @@ template inline void msgpack_schema_pack(Msgpac // Encode as map const_cast(object).msgpack([&](auto&... args) { size_t kv_size = sizeof...(args); + // Calculate the number of entries in our map (half the size of keys + values, plus the typename) packer.pack_map(uint32_t(1 + kv_size / 2)); packer.pack("__typename"); packer.pack(type); + // Pack the map content based on the args to msgpack _schema_pack_map_content(packer, args...); }); } @@ -114,6 +118,7 @@ template inline void msgpack_schema_pack(Msgpac template inline void _msgpack_schema_pack(MsgpackSchemaPacker& packer, const std::string& schema_name) { + // We will pack a size 2 tuple packer.pack_array(2); packer.pack(schema_name); packer.pack_array(sizeof...(Args)); @@ -158,8 +163,10 @@ template inline void msgpack_schema_pack(MsgpackSchemaPacker& packe template inline void msgpack_schema_pack(MsgpackSchemaPacker& packer, std::array const&) { + // We will pack a size 2 tuple packer.pack_array(2); packer.pack("array"); + // That has a size 2 tuple as its 2nd arg packer.pack_array(2); /* param list format for consistency*/ msgpack_schema_pack(packer, T{}); packer.pack(N); From 579cc7796d3305b84c6f5672b19fd42458d42084 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 16 May 2023 23:01:58 -0400 Subject: [PATCH 92/92] Update g1.hpp --- cpp/src/barretenberg/ecc/curves/bn254/g1.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp b/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp index 6d58fd1836..65d0ffaa84 100644 --- a/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp +++ b/cpp/src/barretenberg/ecc/curves/bn254/g1.hpp @@ -21,6 +21,7 @@ typedef group g1; } // namespace barretenberg // specialize the name in msgpack schema generation +// consumed by the typescript schema compiler, helps disambiguate templates inline std::string msgpack_schema_name(barretenberg::g1::affine_element const&) { return "G1AffineElement";