diff --git a/barretenberg/cpp/scripts/stdlib-tests b/barretenberg/cpp/scripts/stdlib-tests index 6cb946112947..75285f483d64 100644 --- a/barretenberg/cpp/scripts/stdlib-tests +++ b/barretenberg/cpp/scripts/stdlib-tests @@ -3,7 +3,6 @@ stdlib_aes128_tests stdlib_blake2s_tests stdlib_blake3s_tests stdlib_ecdsa_tests -stdlib_merkle_tree_tests stdlib_pedersen_commitment_tests stdlib_pedersen_hash_tests stdlib_poseidon2_tests diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 3c996c513dcc..3eea79dfdbf8 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -127,7 +127,7 @@ set(BARRETENBERG_TARGET_OBJECTS $ $ $ - $ + $ $ $ $ diff --git a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt index ae6fa95da27e..145919222651 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt @@ -7,5 +7,9 @@ add_subdirectory(pippenger_bench) add_subdirectory(plonk_bench) add_subdirectory(protogalaxy_bench) add_subdirectory(relations_bench) +add_subdirectory(widgets_bench) +add_subdirectory(poseidon2_bench) +add_subdirectory(merkle_tree_bench) +add_subdirectory(indexed_tree_bench) +add_subdirectory(append_only_tree_bench) add_subdirectory(ultra_bench) -add_subdirectory(widgets_bench) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt new file mode 100644 index 000000000000..7270e7e95305 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(append_only_tree_bench crypto_poseidon2 crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp new file mode 100644 index 000000000000..bde03dcb4ff9 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp @@ -0,0 +1,54 @@ +#include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp" +#include "barretenberg/crypto/merkle_tree/array_store.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include + +using namespace benchmark; +using namespace bb::crypto::merkle_tree; + +using Pedersen = AppendOnlyTree; +using Poseidon2 = AppendOnlyTree; + +const size_t TREE_DEPTH = 32; +const size_t MAX_BATCH_SIZE = 128; + +namespace { +auto& random_engine = bb::numeric::get_randomness(); +} // namespace + +template void perform_batch_insert(TreeType& tree, const std::vector& values) +{ + tree.add_values(values); +} + +template void append_only_tree_bench(State& state) noexcept +{ + const size_t batch_size = size_t(state.range(0)); + const size_t depth = TREE_DEPTH; + + ArrayStore store(depth, 1024 * 1024); + TreeType tree = TreeType(store, depth); + + for (auto _ : state) { + state.PauseTiming(); + std::vector values(batch_size); + for (size_t i = 0; i < batch_size; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + state.ResumeTiming(); + perform_batch_insert(tree, values); + } +} +BENCHMARK(append_only_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(100); +BENCHMARK(append_only_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(1000); + +BENCHMARK_MAIN(); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt index 41d407c044dd..2dfa66906e97 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(goblin_bench ultra_honk eccvm stdlib_recursion stdlib_sha256 stdlib_merkle_tree stdlib_primitives) +barretenberg_module(goblin_bench ultra_honk eccvm stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt new file mode 100644 index 000000000000..998bbdf55def --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(indexed_tree_bench crypto_poseidon2 crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp new file mode 100644 index 000000000000..5e7c0f10abb6 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp @@ -0,0 +1,87 @@ +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp" +#include "barretenberg/crypto/merkle_tree/array_store.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include + +using namespace benchmark; +using namespace bb::crypto::merkle_tree; + +using Poseidon2 = IndexedTree; +using Pedersen = IndexedTree; + +const size_t TREE_DEPTH = 32; +const size_t MAX_BATCH_SIZE = 128; + +namespace { +auto& random_engine = bb::numeric::get_randomness(); +} // namespace + +template +void perform_batch_insert(TreeType& tree, const std::vector& values, bool single_threaded) +{ + tree.add_or_update_values(values, single_threaded); +} + +template void multi_thread_indexed_tree_bench(State& state) noexcept +{ + const size_t batch_size = size_t(state.range(0)); + const size_t depth = TREE_DEPTH; + + ArrayStore store(depth, 1024 * 1024); + TreeType tree = TreeType(store, depth, batch_size); + + for (auto _ : state) { + state.PauseTiming(); + std::vector values(batch_size); + for (size_t i = 0; i < batch_size; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + state.ResumeTiming(); + perform_batch_insert(tree, values, false); + } +} + +template void single_thread_indexed_tree_bench(State& state) noexcept +{ + const size_t batch_size = size_t(state.range(0)); + const size_t depth = TREE_DEPTH; + + ArrayStore store(depth, 1024 * 1024); + TreeType tree = TreeType(store, depth, batch_size); + + for (auto _ : state) { + state.PauseTiming(); + std::vector values(batch_size); + for (size_t i = 0; i < batch_size; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + state.ResumeTiming(); + perform_batch_insert(tree, values, true); + } +} +BENCHMARK(single_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(100); + +BENCHMARK(multi_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(100); + +BENCHMARK(single_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(1000); +BENCHMARK(multi_thread_indexed_tree_bench) + ->Unit(benchmark::kMillisecond) + ->RangeMultiplier(2) + ->Range(2, MAX_BATCH_SIZE) + ->Iterations(1000); + +BENCHMARK_MAIN(); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt index 3f0c92458505..6f622044421a 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(ivc_bench client_ivc stdlib_recursion stdlib_sha256 stdlib_merkle_tree stdlib_primitives) +barretenberg_module(ivc_bench client_ivc stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt new file mode 100644 index 000000000000..f497e924d56a --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(merkle_tree_bench crypto_poseidon2 crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/merkle_tree.bench.cpp similarity index 77% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp rename to barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/merkle_tree.bench.cpp index 49871e881e6f..cdc18ed6c801 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/merkle_tree.bench.cpp @@ -1,15 +1,17 @@ -#include "merkle_tree.hpp" +#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "barretenberg/crypto/merkle_tree/memory_store.hpp" #include "barretenberg/numeric/random/engine.hpp" -#include "hash.hpp" -#include "memory_store.hpp" #include using namespace benchmark; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; + +using TreeType = MerkleTree; namespace { auto& engine = bb::numeric::get_debug_randomness(); -} +} // namespace constexpr size_t DEPTH = 256; constexpr size_t MAX = 4096; @@ -33,7 +35,7 @@ BENCHMARK(hash)->MinTime(5); void update_first_element(State& state) noexcept { MemoryStore store; - MerkleTree db(store, DEPTH); + TreeType db(store, DEPTH); for (auto _ : state) { db.update_element(0, VALUES[1]); @@ -46,7 +48,7 @@ void update_elements(State& state) noexcept for (auto _ : state) { state.PauseTiming(); MemoryStore store; - MerkleTree db(store, DEPTH); + TreeType db(store, DEPTH); state.ResumeTiming(); for (size_t i = 0; i < (size_t)state.range(0); ++i) { db.update_element(i, VALUES[i]); @@ -60,10 +62,10 @@ void update_random_elements(State& state) noexcept for (auto _ : state) { state.PauseTiming(); MemoryStore store; - MerkleTree db(store, DEPTH); + TreeType db(store, DEPTH); for (size_t i = 0; i < (size_t)state.range(0); i++) { state.PauseTiming(); - auto index = MerkleTree::index_t(engine.get_random_uint256()); + auto index = TreeType::index_t(engine.get_random_uint256()); state.ResumeTiming(); db.update_element(index, VALUES[i]); } diff --git a/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/CMakeLists.txt new file mode 100644 index 000000000000..87e6e3d63dd6 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(poseidon2_bench crypto_poseidon2) diff --git a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.bench.cpp b/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/poseidon2.bench.cpp similarity index 57% rename from barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.bench.cpp rename to barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/poseidon2.bench.cpp index 6673734acdc4..4118e4925601 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/poseidon2.bench.cpp @@ -1,4 +1,4 @@ -#include "./poseidon2.hpp" +#include "barretenberg/crypto/poseidon2/poseidon2.hpp" #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" #include @@ -24,4 +24,20 @@ void native_poseidon2_commitment_bench(State& state) noexcept } BENCHMARK(native_poseidon2_commitment_bench)->Arg(10)->Arg(1000)->Arg(10000); +grumpkin::fq poseiden_hash_impl(const grumpkin::fq& x, const grumpkin::fq& y) +{ + std::vector to_hash{ x, y }; + return bb::crypto::Poseidon2::hash(to_hash); +} + +void poseiden_hash_bench(State& state) noexcept +{ + grumpkin::fq x = grumpkin::fq::random_element(); + grumpkin::fq y = grumpkin::fq::random_element(); + for (auto _ : state) { + DoNotOptimize(poseiden_hash_impl(x, y)); + } +} +BENCHMARK(poseiden_hash_bench)->Unit(benchmark::kMillisecond); + BENCHMARK_MAIN(); \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt index 323ad1613b0d..bd67d08ae716 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt @@ -2,5 +2,5 @@ barretenberg_module(ultra_bench ultra_honk stdlib_sha256 stdlib_keccak - stdlib_merkle_tree + crypto_merkle_tree stdlib_recursion) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp index da9c1f2f1084..05942e6afe35 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp +++ b/barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp @@ -2,6 +2,10 @@ #include #include +#include "barretenberg/crypto/merkle_tree/membership.hpp" +#include "barretenberg/crypto/merkle_tree/memory_store.hpp" +#include "barretenberg/crypto/merkle_tree/memory_tree.hpp" +#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp" #include "barretenberg/goblin/mock_circuits.hpp" #include "barretenberg/plonk/composer/standard_composer.hpp" #include "barretenberg/plonk/composer/ultra_composer.hpp" @@ -9,10 +13,6 @@ #include "barretenberg/stdlib/encryption/ecdsa/ecdsa.hpp" #include "barretenberg/stdlib/hash/keccak/keccak.hpp" #include "barretenberg/stdlib/hash/sha256/sha256.hpp" -#include "barretenberg/stdlib/merkle_tree/membership.hpp" -#include "barretenberg/stdlib/merkle_tree/memory_store.hpp" -#include "barretenberg/stdlib/merkle_tree/memory_tree.hpp" -#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp" #include "barretenberg/stdlib/primitives/bool/bool.hpp" #include "barretenberg/stdlib/primitives/curves/secp256k1.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt index 6efc1f824070..5ea623e144e8 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt @@ -9,4 +9,5 @@ add_subdirectory(schnorr) add_subdirectory(sha256) add_subdirectory(ecdsa) add_subdirectory(aes128) -add_subdirectory(poseidon2) \ No newline at end of file +add_subdirectory(poseidon2) +add_subdirectory(merkle_tree) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt new file mode 100644 index 000000000000..2c48323594be --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt @@ -0,0 +1 @@ +barretenberg_module(crypto_merkle_tree stdlib_primitives stdlib_blake3s stdlib_pedersen_hash) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp new file mode 100644 index 000000000000..7170a9292ea7 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp @@ -0,0 +1,195 @@ +#pragma once +#include "../hash_path.hpp" + +namespace bb::crypto::merkle_tree { + +using namespace bb; + +typedef uint256_t index_t; + +/** + * @brief Implements a simple append-only merkle tree + * Accepts template argument of the type of store backing the tree and the hashing policy + * + */ +template class AppendOnlyTree { + public: + AppendOnlyTree(Store& store, size_t depth, uint8_t tree_id = 0); + AppendOnlyTree(AppendOnlyTree const& other) = delete; + AppendOnlyTree(AppendOnlyTree&& other) = delete; + virtual ~AppendOnlyTree(); + + /** + * @brief Adds a single value to the end of the tree + */ + virtual fr add_value(const fr& value); + + /** + * @brief Adds the given set of values to the end of the tree + */ + virtual fr add_values(const std::vector& values); + + /** + * @brief Returns the index of the right-most populated leaf in the tree + */ + index_t size() const; + + /** + * @brief Returns the root of the tree + */ + fr root() const; + + /** + * @brief Returns the depth of the tree + */ + size_t depth() const; + + /** + * @brief Returns the hash path from the leaf at the given index to the root + */ + fr_hash_path get_hash_path(const index_t& index) const; + + protected: + fr get_element_or_zero(size_t level, const index_t& index) const; + + void write_node(size_t level, const index_t& index, const fr& value); + std::pair read_node(size_t level, const index_t& index) const; + + Store& store_; + size_t depth_; + uint8_t tree_id_; + std::vector zero_hashes_; + fr root_; + index_t size_; +}; + +template +AppendOnlyTree::AppendOnlyTree(Store& store, size_t depth, uint8_t tree_id) + : store_(store) + , depth_(depth) + , tree_id_(tree_id) +{ + zero_hashes_.resize(depth + 1); + + // Create the zero hashes for the tree + auto current = HashingPolicy::zero_hash(); + for (size_t i = depth; i > 0; --i) { + zero_hashes_[i] = current; + current = HashingPolicy::hash_pair(current, current); + } + zero_hashes_[0] = current; + root_ = current; +} + +template AppendOnlyTree::~AppendOnlyTree() {} + +template index_t AppendOnlyTree::size() const +{ + return size_; +} + +template fr AppendOnlyTree::root() const +{ + return root_; +} + +template size_t AppendOnlyTree::depth() const +{ + return depth_; +} + +template +fr_hash_path AppendOnlyTree::get_hash_path(const index_t& index) const +{ + fr_hash_path path; + index_t current_index = index; + + for (size_t level = depth_; level > 0; --level) { + bool is_right = bool(current_index & 0x01); + fr right_value = + is_right ? get_element_or_zero(level, current_index) : get_element_or_zero(level, current_index + 1); + fr left_value = + is_right ? get_element_or_zero(level, current_index - 1) : get_element_or_zero(level, current_index); + path.push_back(std::make_pair(left_value, right_value)); + current_index >>= 1; + } + return path; +} + +template fr AppendOnlyTree::add_value(const fr& value) +{ + return add_values(std::vector{ value }); +} + +template +fr AppendOnlyTree::add_values(const std::vector& values) +{ + index_t index = size(); + size_t number_to_insert = values.size(); + size_t level = depth_; + std::vector hashes = values; + + // Add the values at the leaf nodes of the tree + for (size_t i = 0; i < number_to_insert; ++i) { + write_node(level, index + i, hashes[i]); + } + + // Hash the values as a sub tree and insert them + while (number_to_insert > 1) { + number_to_insert >>= 1; + index >>= 1; + --level; + for (size_t i = 0; i < number_to_insert; ++i) { + hashes[i] = HashingPolicy::hash_pair(hashes[i * 2], hashes[i * 2 + 1]); + write_node(level, index + i, hashes[i]); + } + } + + // Hash from the root of the sub-tree to the root of the overall tree + fr new_hash = hashes[0]; + while (level > 0) { + bool is_right = bool(index & 0x01); + fr left_hash = is_right ? get_element_or_zero(level, index - 1) : new_hash; + fr right_hash = is_right ? new_hash : get_element_or_zero(level, index + 1); + new_hash = HashingPolicy::hash_pair(left_hash, right_hash); + index >>= 1; + --level; + write_node(level, index, new_hash); + } + size_ += values.size(); + root_ = new_hash; + return root_; +} + +template +fr AppendOnlyTree::get_element_or_zero(size_t level, const index_t& index) const +{ + const std::pair read_data = read_node(level, index); + if (read_data.first) { + return read_data.second; + } + ASSERT(level > 0 && level < zero_hashes_.size()); + return zero_hashes_[level]; +} + +template +void AppendOnlyTree::write_node(size_t level, const index_t& index, const fr& value) +{ + std::vector buf; + write(buf, value); + store_.put(level, size_t(index), buf); +} + +template +std::pair AppendOnlyTree::read_node(size_t level, const index_t& index) const +{ + std::vector buf; + bool available = store_.get(level, size_t(index), buf); + if (!available) { + return std::make_pair(false, fr::zero()); + } + fr value = from_buffer(buf, 0); + return std::make_pair(true, value); +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp new file mode 100644 index 000000000000..e25f53ba1ca8 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.test.cpp @@ -0,0 +1,127 @@ +#include "append_only_tree.hpp" +#include "../array_store.hpp" +#include "../memory_tree.hpp" +#include "barretenberg/common/streams.hpp" +#include "barretenberg/common/test.hpp" +#include "barretenberg/numeric/random/engine.hpp" + +using namespace bb; +using namespace bb::crypto::merkle_tree; + +namespace { +auto& engine = numeric::get_debug_randomness(); +auto& random_engine = numeric::get_randomness(); +} // namespace + +const size_t NUM_VALUES = 1024; +static std::vector VALUES = []() { + std::vector values(NUM_VALUES); + for (size_t i = 0; i < NUM_VALUES; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + return values; +}(); + +inline void print_tree(const size_t depth, std::vector hashes, std::string const& msg) +{ + info("\n", msg); + size_t offset = 0; + for (size_t i = 0; i < depth; i++) { + info("i = ", i); + size_t layer_size = (1UL << (depth - i)); + for (size_t j = 0; j < layer_size; j++) { + info("j = ", j, ": ", hashes[offset + j]); + } + offset += layer_size; + } +} + +TEST(stdlib_append_only_tree, can_create) +{ + constexpr size_t depth = 10; + ArrayStore store(depth); + AppendOnlyTree tree(store, depth); + MemoryTree memdb(depth); + + EXPECT_EQ(tree.size(), 0); + EXPECT_EQ(tree.root(), memdb.root()); +} + +TEST(stdlib_append_only_tree, can_add_value) +{ + constexpr size_t depth = 10; + ArrayStore store(depth); + AppendOnlyTree tree(store, depth); + MemoryTree memdb(depth); + + EXPECT_EQ(tree.size(), 0); + EXPECT_EQ(tree.root(), memdb.root()); + + memdb.update_element(0, VALUES[0]); + tree.add_value(VALUES[0]); + + EXPECT_EQ(tree.root(), memdb.root()); + EXPECT_EQ(tree.get_hash_path(0), memdb.get_hash_path(0)); +} + +TEST(stdlib_append_only_tree, test_size) +{ + constexpr size_t depth = 10; + ArrayStore store(depth); + AppendOnlyTree tree(store, depth); + + EXPECT_EQ(tree.size(), 0ULL); + + // Add a new non-zero leaf at index 0. + tree.add_value(30); + EXPECT_EQ(tree.size(), 1ULL); + + // Add second. + tree.add_value(10); + EXPECT_EQ(tree.size(), 2ULL); + + // Add third. + tree.add_value(20); + EXPECT_EQ(tree.size(), 3ULL); + + // Add forth but with same value. + tree.add_value(20); + EXPECT_EQ(tree.size(), 4ULL); +} + +TEST(stdlib_append_only_tree, can_add_multiple_values) +{ + constexpr size_t depth = 10; + ArrayStore store(depth); + AppendOnlyTree tree(store, depth); + MemoryTree memdb(depth); + + for (size_t i = 0; i < NUM_VALUES; ++i) { + fr mock_root = memdb.update_element(i, VALUES[i]); + fr tree_root = tree.add_value(VALUES[i]); + EXPECT_EQ(mock_root, tree_root); + + EXPECT_EQ(memdb.get_hash_path(0), tree.get_hash_path(0)); + EXPECT_EQ(memdb.get_hash_path(i), tree.get_hash_path(i)); + } +} + +TEST(stdlib_append_only_tree, can_be_filled) +{ + constexpr size_t depth = 3; + ArrayStore store(depth); + AppendOnlyTree tree(store, depth); + MemoryTree memdb(depth); + + EXPECT_EQ(tree.size(), 0); + EXPECT_EQ(tree.root(), memdb.root()); + + for (size_t i = 0; i < 8; i++) { + memdb.update_element(i, VALUES[i]); + tree.add_value(VALUES[i]); + } + + EXPECT_EQ(tree.root(), memdb.root()); + EXPECT_EQ(tree.get_hash_path(0), memdb.get_hash_path(0)); + EXPECT_EQ(tree.get_hash_path(7), memdb.get_hash_path(7)); +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp new file mode 100644 index 000000000000..ade8472b1b91 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/array_store.hpp @@ -0,0 +1,37 @@ +#pragma once +#include "barretenberg/stdlib/primitives/field/field.hpp" + +namespace bb::crypto::merkle_tree { + +/** + * @brief A very basic 2-d array for use as a backing store for merkle trees. + * Can store up to 'indices' nodes per row and 'levels' rows. + */ +class ArrayStore { + + public: + ArrayStore(size_t levels, size_t indices = 1024) + : map_(std::vector>>>( + levels + 1, + std::vector>>( + indices, std::pair>(false, std::vector())))) + {} + ~ArrayStore() {} + + void put(size_t level, size_t index, const std::vector& data) + { + map_[level][index] = std::make_pair(true, data); + } + bool get(size_t level, size_t index, std::vector& data) const + { + const std::pair>& slot = map_[level][index]; + if (slot.first) { + data = slot.second; + } + return slot.first; + } + + private: + std::vector>>> map_; +}; +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.hpp similarity index 73% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.hpp index 4e5188206804..79a0b7e09c36 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.hpp @@ -3,12 +3,32 @@ #include "barretenberg/crypto/blake2s/blake2s.hpp" #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/crypto/pedersen_hash/pedersen.hpp" +#include "barretenberg/crypto/poseidon2/poseidon2.hpp" #include "barretenberg/stdlib/hash/blake2s/blake2s.hpp" #include "barretenberg/stdlib/hash/pedersen/pedersen.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" #include -namespace bb::stdlib::merkle_tree { +namespace bb::crypto::merkle_tree { + +struct PedersenHashPolicy { + static fr hash(const std::vector& inputs) { return crypto::pedersen_hash::hash(inputs); } + + static fr hash_pair(const fr& lhs, const fr& rhs) { return hash(std::vector({ lhs, rhs })); } + + static fr zero_hash() { return fr::zero(); } +}; + +struct Poseidon2HashPolicy { + static fr hash(const std::vector& inputs) + { + return bb::crypto::Poseidon2::hash(inputs); + } + + static fr hash_pair(const fr& lhs, const fr& rhs) { return hash(std::vector({ lhs, rhs })); } + + static fr zero_hash() { return fr::zero(); } +}; inline bb::fr hash_pair_native(bb::fr const& lhs, bb::fr const& rhs) { @@ -63,4 +83,4 @@ inline std::vector compute_tree_native(std::vector const& input) return tree; } -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.test.cpp similarity index 75% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.test.cpp index 2e420fb15383..a8eeec7cc697 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash.test.cpp @@ -2,29 +2,29 @@ #include "memory_tree.hpp" #include +#include "barretenberg/crypto/merkle_tree/membership.hpp" #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp" -#include "barretenberg/stdlib/merkle_tree/membership.hpp" using namespace bb; -using namespace bb::stdlib; +using namespace bb::crypto; using Builder = UltraCircuitBuilder; -using field_ct = field_t; -using witness_ct = witness_t; +using field_ct = bb::stdlib::field_t; +using witness_ct = bb::stdlib::witness_t; -TEST(stdlib_merkle_tree_hash, hash_native_vs_circuit) +TEST(crypto_merkle_tree_hash, hash_native_vs_circuit) { fr x = uint256_t(0x5ec473eb273a8011, 0x50160109385471ca, 0x2f3095267e02607d, 0x02586f4a39e69b86); Builder builder = Builder(); witness_ct y = witness_ct(&builder, x); - field_ct z = pedersen_hash::hash({ y, y }); + field_ct z = bb::stdlib::pedersen_hash::hash({ y, y }); auto zz = merkle_tree::hash_pair_native(x, x); EXPECT_EQ(z.get_value(), zz); } -TEST(stdlib_merkle_tree_hash, compute_tree_root_native_vs_circuit) +TEST(crypto_merkle_tree_hash, compute_tree_root_native_vs_circuit) { Builder builder = Builder(); std::vector inputs; @@ -42,10 +42,10 @@ TEST(stdlib_merkle_tree_hash, compute_tree_root_native_vs_circuit) EXPECT_EQ(z.get_value(), zz); } -TEST(stdlib_merkle_tree_hash, compute_tree_native) +TEST(crypto_merkle_tree_hash, compute_tree_native) { constexpr size_t depth = 2; - merkle_tree::MemoryTree mem_tree(depth); + merkle_tree::MemoryTree mem_tree(depth); std::vector leaves; for (size_t i = 0; i < (size_t(1) << depth); i++) { diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash_path.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash_path.hpp similarity index 83% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash_path.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash_path.hpp index f661b290c488..fb9130bda2e2 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/hash_path.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/hash_path.hpp @@ -4,13 +4,12 @@ #include #include -namespace bb::stdlib::merkle_tree { - -using namespace bb; +namespace bb::crypto::merkle_tree { +using fr = bb::stdlib::fr; using fr_hash_path = std::vector>; using fr_sibling_path = std::vector; -template using hash_path = std::vector, field_t>>; +template using hash_path = std::vector, bb::stdlib::field_t>>; inline fr_hash_path get_new_hash_path(fr_hash_path const& old_path, uint128_t index, fr const& value) { @@ -42,7 +41,8 @@ template inline hash_path create_witness_hash_path(Ctx& ctx, { hash_path result; std::transform(input.begin(), input.end(), std::back_inserter(result), [&](auto const& v) { - return std::make_pair(field_t(witness_t(&ctx, v.first)), field_t(witness_t(&ctx, v.second))); + return std::make_pair(bb::stdlib::field_t(bb::stdlib::witness_t(&ctx, v.first)), + bb::stdlib::field_t(bb::stdlib::witness_t(&ctx, v.second))); }); return result; } @@ -61,13 +61,13 @@ inline fr zero_hash_at_height(size_t height) return current; } -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree // We add to std namespace as fr_hash_path is actually a std::vector, and this is the only way // to achieve effective ADL. namespace std { template -inline std::ostream& operator<<(std::ostream& os, bb::stdlib::merkle_tree::hash_path const& path) +inline std::ostream& operator<<(std::ostream& os, bb::crypto::merkle_tree::hash_path const& path) { os << "[\n"; for (size_t i = 0; i < path.size(); ++i) { @@ -77,7 +77,7 @@ inline std::ostream& operator<<(std::ostream& os, bb::stdlib::merkle_tree::hash_ return os; } -inline std::ostream& operator<<(std::ostream& os, bb::stdlib::merkle_tree::fr_hash_path const& path) +inline std::ostream& operator<<(std::ostream& os, bb::crypto::merkle_tree::fr_hash_path const& path) { os << "[\n"; for (size_t i = 0; i < path.size(); ++i) { diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/index.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/index.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/index.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/index.hpp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp new file mode 100644 index 000000000000..bd5e9859cbec --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include "barretenberg/stdlib/primitives/field/field.hpp" + +namespace bb::crypto::merkle_tree { + +typedef uint256_t index_t; + +struct indexed_leaf { + fr value; + index_t nextIndex; + fr nextValue; + + bool operator==(indexed_leaf const&) const = default; + + std::ostream& operator<<(std::ostream& os) + { + os << "value = " << value << "\nnextIdx = " << nextIndex << "\nnextVal = " << nextValue; + return os; + } + + std::vector get_hash_inputs() const { return std::vector({ value, nextIndex, nextValue }); } +}; + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp new file mode 100644 index 000000000000..3afb64e91634 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp @@ -0,0 +1,379 @@ +#pragma once +#include "../../../common/thread.hpp" +#include "../append_only_tree/append_only_tree.hpp" +#include "../hash.hpp" +#include "../hash_path.hpp" +#include "indexed_leaf.hpp" + +namespace bb::crypto::merkle_tree { + +typedef uint256_t index_t; + +/** + * @brief Used in parallel insertions in the the IndexedTree. Workers signal to other following workes as they move up + * the level of the tree. + * + */ +class LevelSignal { + public: + LevelSignal(size_t initial_level) + : signal_(initial_level){}; + ~LevelSignal(){}; + LevelSignal(const LevelSignal& other) + : signal_(other.signal_.load()) + {} + LevelSignal(const LevelSignal&& other) = delete; + + /** + * @brief Causes the thread to wait until the required level has been signalled + * @param level The required level + * + */ + void wait_for_level(size_t level) + { + size_t current_level = signal_.load(); + while (current_level > level) { + signal_.wait(current_level); + current_level = signal_.load(); + } + } + + /** + * @brief Signals that the given level has been passed + * @param level The level to be signalled + * + */ + void signal_level(size_t level) + { + signal_.store(level); + signal_.notify_all(); + } + + private: + std::atomic signal_; +}; + +/** + * @brief Implements a parallelised batch insertion indexed tree + * Accepts template argument of the type of store backing the tree, the type of store containing the leaves and the + * hashing policy + * + */ +template +class IndexedTree : public AppendOnlyTree { + public: + IndexedTree(Store& store, size_t depth, size_t initial_size = 1, uint8_t tree_id = 0); + IndexedTree(IndexedTree const& other) = delete; + IndexedTree(IndexedTree&& other) = delete; + ~IndexedTree(); + + /** + * @brief Adds or updates a single values in the tree (updates not currently supported) + * @param value The value to be added or updated + * @returns The 'previous' hash paths of all updated values + */ + fr_hash_path add_or_update_value(const fr& value); + + /** + * @brief Adds or updates the given set of values in the tree (updates not currently supported) + * @param values The values to be added or updated + * @param no_multithreading Performs single threaded insertion, just used whilst prototyping and benchmarking + * @returns The 'previous' hash paths of all updated values + */ + std::vector add_or_update_values(const std::vector& values, bool no_multithreading = false); + + /** + * @brief Adds or updates a single value without returning the previous hash path + * @param value The value to be added or updated + * @returns The new root of the tree + */ + fr add_value(const fr& value) override; + + /** + * @brief Adds or updates the given set of values without returning the previous hash paths + * @param values The values to be added or updated + * @returns The new root of the tree + */ + fr add_values(const std::vector& values) override; + + indexed_leaf get_leaf(const index_t& index); + + using AppendOnlyTree::get_hash_path; + using AppendOnlyTree::root; + using AppendOnlyTree::depth; + + private: + fr update_leaf_and_hash_to_root(const index_t& index, const indexed_leaf& leaf); + fr update_leaf_and_hash_to_root(const index_t& index, + const indexed_leaf& leaf, + LevelSignal& leader, + LevelSignal& follower, + fr_hash_path& previous_hash_path); + fr append_subtree(const index_t& start_index); + + using AppendOnlyTree::get_element_or_zero; + using AppendOnlyTree::write_node; + using AppendOnlyTree::read_node; + + private: + using AppendOnlyTree::store_; + using AppendOnlyTree::zero_hashes_; + using AppendOnlyTree::depth_; + using AppendOnlyTree::tree_id_; + using AppendOnlyTree::root_; + LeavesStore leaves_; +}; + +template +IndexedTree::IndexedTree(Store& store, + size_t depth, + size_t initial_size, + uint8_t tree_id) + : AppendOnlyTree(store, depth, tree_id) +{ + ASSERT(initial_size > 0); + zero_hashes_.resize(depth + 1); + + // Create the zero hashes for the tree + indexed_leaf zero_leaf{ 0, 0, 0 }; + auto current = HashingPolicy::hash(zero_leaf.get_hash_inputs()); + for (size_t i = depth; i > 0; --i) { + zero_hashes_[i] = current; + current = HashingPolicy::hash_pair(current, current); + } + zero_hashes_[0] = current; + // Inserts the initial set of leaves as a chain in incrementing value order + for (size_t i = 0; i < initial_size; ++i) { + // Insert the zero leaf to the `leaves` and also to the tree at index 0. + indexed_leaf initial_leaf = indexed_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }; + leaves_.append_leaf(initial_leaf); + } + + // Points the last leaf back to the first + leaves_.set_at_index( + initial_size - 1, + indexed_leaf{ .value = leaves_.get_leaf(initial_size - 1).value, .nextIndex = 0, .nextValue = 0 }, + false); + append_subtree(0); +} + +template +IndexedTree::~IndexedTree() +{} + +template +indexed_leaf IndexedTree::get_leaf(const index_t& index) +{ + return leaves_.get_leaf(index); +} + +template +fr IndexedTree::add_value(const fr& value) +{ + return add_values(std::vector{ value }); +} + +template +fr IndexedTree::add_values(const std::vector& values) +{ + add_or_update_values(values); + return root(); +} + +template +fr_hash_path IndexedTree::add_or_update_value(const fr& value) +{ + return add_or_update_values(std::vector{ value })[0]; +} + +template +std::vector IndexedTree::add_or_update_values( + const std::vector& values, bool no_multithreading) +{ + // The first thing we do is sort the values into descending order but maintain knowledge of their orignal order + struct { + bool operator()(const std::pair& a, const std::pair& b) const + { + return uint256_t(a.first) > uint256_t(b.first); + } + } comp; + std::vector> values_sorted(values.size()); + for (size_t i = 0; i < values.size(); ++i) { + values_sorted[i] = std::make_pair(values[i], i); + } + std::sort(values_sorted.begin(), values_sorted.end(), comp); + + // Now that we have the sorted values we need to identify the leaves that need updating. + // This is performed sequentially and is stored in this 'leaf_insertion' struct + struct leaf_insertion { + index_t low_leaf_index; + indexed_leaf low_leaf; + }; + + std::vector insertions(values.size()); + index_t old_size = leaves_.get_size(); + + for (size_t i = 0; i < values_sorted.size(); ++i) { + fr value = values_sorted[i].first; + index_t index_of_new_leaf = index_t(values_sorted[i].second) + old_size; + + // This gives us the leaf that need updating + index_t current; + bool is_already_present; + std::tie(is_already_present, current) = leaves_.find_low_value(values_sorted[i].first); + indexed_leaf current_leaf = leaves_.get_leaf(current); + + indexed_leaf new_leaf = + indexed_leaf{ .value = value, .nextIndex = current_leaf.nextIndex, .nextValue = current_leaf.nextValue }; + + // We only handle new values being added. We don't yet handle values being updated + if (!is_already_present) { + // Update the current leaf to point it to the new leaf + current_leaf.nextIndex = index_of_new_leaf; + current_leaf.nextValue = value; + + leaves_.set_at_index(current, current_leaf, false); + leaves_.set_at_index(index_of_new_leaf, new_leaf, true); + } + + // Capture the index and value of the updated 'low' leaf + leaf_insertion& insertion = insertions[i]; + insertion.low_leaf_index = current; + insertion.low_leaf = indexed_leaf{ .value = current_leaf.value, + .nextIndex = current_leaf.nextIndex, + .nextValue = current_leaf.nextValue }; + } + + // We now kick off multiple workers to perform the low leaf updates + // We create set of signals to coordinate the workers as the move up the tree + std::vector paths(insertions.size()); + std::vector signals; + // The first signal is set to 0. This ensure the first worker up the tree is not impeded + signals.emplace_back(size_t(0)); + // Workers will follow their leaders up the tree, being trigger by the signal in front of them + for (size_t i = 0; i < insertions.size(); ++i) { + signals.emplace_back(size_t(1 + depth_)); + } + + if (no_multithreading) { + // Execute the jobs in series + for (size_t i = 0; i < insertions.size(); ++i) { + leaf_insertion& insertion = insertions[i]; + update_leaf_and_hash_to_root( + insertion.low_leaf_index, insertion.low_leaf, signals[i], signals[i + 1], paths[i]); + } + } else { + // Execute the jobs in parallel + parallel_for(insertions.size(), [&](size_t i) { + leaf_insertion& insertion = insertions[i]; + update_leaf_and_hash_to_root( + insertion.low_leaf_index, insertion.low_leaf, signals[i], signals[i + 1], paths[i]); + }); + } + + // Now that we have updated all of the low leaves, we insert the new leaves as a subtree at the end + root_ = append_subtree(old_size); + + return paths; +} + +template +fr IndexedTree::update_leaf_and_hash_to_root(const index_t& leaf_index, + const indexed_leaf& leaf) +{ + LevelSignal leader(0); + LevelSignal follower(0); + fr_hash_path hash_path; + return update_leaf_and_hash_to_root(leaf_index, leaf, leader, follower, hash_path); +} + +template +fr IndexedTree::update_leaf_and_hash_to_root(const index_t& leaf_index, + const indexed_leaf& leaf, + LevelSignal& leader, + LevelSignal& follower, + fr_hash_path& previous_hash_path) +{ + // We are a worker at a specific leaf index. + // We are going to move up the tree and at each node/level: + // 1. Wait for the level above to become 'signalled' as clear for us to write into + // 2. Read the node and it's sibling + // 3. Write the new node value + index_t index = leaf_index; + size_t level = depth_; + fr new_hash = HashingPolicy::hash(leaf.get_hash_inputs()); + + // Wait until we see that our leader has cleared 'depth_ - 1' (i.e. the level above the leaves that we are about to + // write into) this ensures that our leader is not still reading the leaves + size_t leader_level = depth_ - 1; + leader.wait_for_level(leader_level); + + // Extract the value of the leaf node and it's sibling + bool is_right = bool(index & 0x01); + // extract the current leaf hash values for the previous hash path + fr current_right_value = get_element_or_zero(level, index + (is_right ? 0 : 1)); + fr current_left_value = get_element_or_zero(level, is_right ? (index - 1) : index); + previous_hash_path.push_back(std::make_pair(current_left_value, current_right_value)); + + // Write the new leaf hash in place + write_node(level, index, new_hash); + // Signal that this level has been written + follower.signal_level(level); + + while (level > 0) { + if (level > 1) { + // Level is > 1. Therefore we need to wait for our leader to have written to the level above meaning we can + // read from it + size_t level_to_read = level - 1; + leader_level = level_to_read; + + leader.wait_for_level(leader_level); + + // Now read the node and it's sibling + index_t index_of_node_above = index >> 1; + bool node_above_is_right = bool(index_of_node_above & 0x01); + fr above_right_value = + get_element_or_zero(level_to_read, index_of_node_above + (node_above_is_right ? 0 : 1)); + fr above_left_value = get_element_or_zero( + level_to_read, node_above_is_right ? (index_of_node_above - 1) : index_of_node_above); + previous_hash_path.push_back(std::make_pair(above_left_value, above_right_value)); + } + + // Now that we have extracted the hash path from the row above + // we can compute the new hash at that level and write it + is_right = bool(index & 0x01); + fr new_right_value = is_right ? new_hash : get_element_or_zero(level, index + 1); + fr new_left_value = is_right ? get_element_or_zero(level, index - 1) : new_hash; + new_hash = HashingPolicy::hash_pair(new_left_value, new_right_value); + index >>= 1; + --level; + if (level > 0) { + // Before we write we need to ensure that our leader has already written to the row above it + // otherwise it could still be reading from this level + leader_level = level - 1; + leader.wait_for_level(leader_level); + } + + // Write this node and signal that it is done + write_node(level, index, new_hash); + follower.signal_level(level); + } + return new_hash; +} + +template +fr IndexedTree::append_subtree(const index_t& start_index) +{ + index_t index = start_index; + size_t number_to_insert = size_t(index_t(leaves_.get_size()) - index); + std::vector hashes_to_append = std::vector(number_to_insert); + + for (size_t i = 0; i < number_to_insert; ++i) { + index_t index_to_insert = index + i; + hashes_to_append[i] = HashingPolicy::hash(leaves_.get_leaf(size_t(index_to_insert)).get_hash_inputs()); + } + + return AppendOnlyTree::add_values(hashes_to_append); +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp new file mode 100644 index 000000000000..d1dc13d95e52 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -0,0 +1,331 @@ +#include "indexed_tree.hpp" +#include "../array_store.hpp" +#include "../hash.hpp" +#include "../nullifier_tree/nullifier_memory_tree.hpp" +#include "barretenberg/common/streams.hpp" +#include "barretenberg/common/test.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include "leaves_cache.hpp" + +using namespace bb; +using namespace bb::crypto::merkle_tree; + +using HashPolicy = Poseidon2HashPolicy; + +namespace { +auto& engine = numeric::get_debug_randomness(); +auto& random_engine = numeric::get_randomness(); +} // namespace + +const size_t NUM_VALUES = 1024; +static std::vector VALUES = []() { + std::vector values(NUM_VALUES); + for (size_t i = 0; i < NUM_VALUES; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + return values; +}(); + +TEST(stdlib_indexed_tree, can_create) +{ + ArrayStore store(10); + IndexedTree tree = IndexedTree(store, 10); + EXPECT_EQ(tree.size(), 1ULL); + + NullifierMemoryTree memdb(10); + EXPECT_EQ(memdb.root(), tree.root()); +} + +TEST(stdlib_indexed_tree, test_size) +{ + ArrayStore store(32); + auto db = IndexedTree(store, 32); + + // We assume that the first leaf is already filled with (0, 0, 0). + EXPECT_EQ(db.size(), 1ULL); + + // Add a new non-zero leaf at index 1. + db.add_value(30); + EXPECT_EQ(db.size(), 2ULL); + + // Add third. + db.add_value(10); + EXPECT_EQ(db.size(), 3ULL); + + // Add forth. + db.add_value(20); + EXPECT_EQ(db.size(), 4ULL); +} + +TEST(stdlib_indexed_tree, test_get_hash_path) +{ + NullifierMemoryTree memdb(10); + + ArrayStore store(10); + auto db = IndexedTree(store, 10); + + EXPECT_EQ(memdb.root(), db.root()); + + EXPECT_EQ(memdb.get_hash_path(0), db.get_hash_path(0)); + + memdb.update_element(VALUES[512]); + db.add_value(VALUES[512]); + + EXPECT_EQ(db.get_hash_path(0), memdb.get_hash_path(0)); + + for (size_t i = 0; i < 512; ++i) { + memdb.update_element(VALUES[i]); + db.add_value(VALUES[i]); + } + + EXPECT_EQ(db.get_hash_path(512), memdb.get_hash_path(512)); +} + +TEST(stdlib_indexed_tree, test_batch_insert) +{ + const size_t batch_size = 16; + const size_t num_batches = 16; + size_t depth = 10; + NullifierMemoryTree memdb(depth, batch_size); + + ArrayStore store1(depth); + IndexedTree tree1 = + IndexedTree(store1, depth, batch_size); + + ArrayStore store2(depth); + IndexedTree tree2 = + IndexedTree(store2, depth, batch_size); + + EXPECT_EQ(memdb.root(), tree1.root()); + EXPECT_EQ(tree1.root(), tree2.root()); + + EXPECT_EQ(memdb.get_hash_path(0), tree1.get_hash_path(0)); + EXPECT_EQ(tree1.get_hash_path(0), tree2.get_hash_path(0)); + + EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); + EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); + + for (size_t i = 0; i < num_batches; i++) { + std::vector batch; + std::vector memory_tree_hash_paths; + for (size_t j = 0; j < batch_size; j++) { + batch.push_back(fr(random_engine.get_random_uint256())); + fr_hash_path path = memdb.update_element(batch[j]); + memory_tree_hash_paths.push_back(path); + } + std::vector tree1_hash_paths = tree1.add_or_update_values(batch, true); + std::vector tree2_hash_paths = tree2.add_or_update_values(batch); + EXPECT_EQ(memdb.root(), tree1.root()); + EXPECT_EQ(tree1.root(), tree2.root()); + + EXPECT_EQ(memdb.get_hash_path(0), tree1.get_hash_path(0)); + EXPECT_EQ(tree1.get_hash_path(0), tree2.get_hash_path(0)); + + EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); + EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); + + for (size_t j = 0; j < batch_size; j++) { + EXPECT_EQ(tree1_hash_paths[j], tree2_hash_paths[j]); + // EXPECT_EQ(tree1_hash_paths[j], memory_tree_hash_paths[j]); + } + } +} + +fr hash_leaf(const indexed_leaf& leaf) +{ + return HashPolicy::hash(leaf.get_hash_inputs()); +} + +bool check_hash_path(const fr& root, const fr_hash_path& path, const indexed_leaf& leaf_value, const size_t idx) +{ + auto current = hash_leaf(leaf_value); + size_t depth_ = path.size(); + size_t index = idx; + for (size_t i = 0; i < depth_; ++i) { + fr left = (index & 1) ? path[i].first : current; + fr right = (index & 1) ? current : path[i].second; + current = HashPolicy::hash_pair(left, right); + index >>= 1; + } + return current == root; +} + +TEST(stdlib_indexed_tree, test_indexed_memory) +{ + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + ArrayStore store(depth); + IndexedTree tree = + IndexedTree(store, depth, 1); + + /** + * Intial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 0 0 0 0 0 0 0 + * nextIdx 0 0 0 0 0 0 0 0 + * nextVal 0 0 0 0 0 0 0 0 + */ + indexed_leaf zero_leaf = { 0, 0, 0 }; + EXPECT_EQ(tree.size(), 1); + EXPECT_EQ(tree.get_leaf(0), zero_leaf); + + /** + * Add new value 30: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 30 0 0 0 0 0 0 0 + */ + tree.add_value(30); + EXPECT_EQ(tree.size(), 2); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 1, 30 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + + /** + * Add new value 10: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 0 0 0 0 0 + * nextIdx 2 0 1 0 0 0 0 0 + * nextVal 10 0 30 0 0 0 0 0 + */ + tree.add_value(10); + EXPECT_EQ(tree.size(), 3); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 1, 30 })); + + /** + * Add new value 20: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + */ + tree.add_value(20); + EXPECT_EQ(tree.size(), 4); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 3, 20 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(3)), hash_leaf({ 20, 1, 30 })); + + // Adding the same value must not affect anything + // tree.update_element(20); + // EXPECT_EQ(tree.get_leaves().size(), 4); + // EXPECT_EQ(tree.get_leaves()[0], hash_leaf({ 0, 2, 10 })); + // EXPECT_EQ(tree.get_leaves()[1], hash_leaf({ 30, 0, 0 })); + // EXPECT_EQ(tree.get_leaves()[2], hash_leaf({ 10, 3, 20 })); + // EXPECT_EQ(tree.get_leaves()[3], hash_leaf({ 20, 1, 30 })); + + /** + * Add new value 50: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 50 0 0 0 + * nextIdx 2 4 3 1 0 0 0 0 + * nextVal 10 50 20 30 0 0 0 0 + */ + tree.add_value(50); + EXPECT_EQ(tree.size(), 5); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 4, 50 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 3, 20 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(3)), hash_leaf({ 20, 1, 30 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(4)), hash_leaf({ 50, 0, 0 })); + + // Manually compute the node values + auto e000 = hash_leaf(tree.get_leaf(0)); + auto e001 = hash_leaf(tree.get_leaf(1)); + auto e010 = hash_leaf(tree.get_leaf(2)); + auto e011 = hash_leaf(tree.get_leaf(3)); + auto e100 = hash_leaf(tree.get_leaf(4)); + auto e101 = hash_leaf({ 0, 0, 0 }); + auto e110 = hash_leaf({ 0, 0, 0 }); + auto e111 = hash_leaf({ 0, 0, 0 }); + + auto e00 = HashPolicy::hash_pair(e000, e001); + auto e01 = HashPolicy::hash_pair(e010, e011); + auto e10 = HashPolicy::hash_pair(e100, e101); + auto e11 = HashPolicy::hash_pair(e110, e111); + + auto e0 = HashPolicy::hash_pair(e00, e01); + auto e1 = HashPolicy::hash_pair(e10, e11); + auto root = HashPolicy::hash_pair(e0, e1); + + // Check the hash path at index 2 and 3 + // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) + fr_hash_path expected = { + std::make_pair(e000, e001), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(0), expected); + EXPECT_EQ(tree.get_hash_path(1), expected); + fr_hash_path expected2 = { + std::make_pair(e010, e011), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(2), expected2); + EXPECT_EQ(tree.get_hash_path(3), expected2); + EXPECT_EQ(tree.root(), root); + + // Check the hash path at index 6 and 7 + expected = { + std::make_pair(e110, e111), + std::make_pair(e10, e11), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(6), expected); + EXPECT_EQ(tree.get_hash_path(7), expected); +} + +TEST(stdlib_indexed_tree, test_indexed_tree) +{ + // Create a depth-8 indexed merkle tree + constexpr size_t depth = 8; + ArrayStore store(depth); + IndexedTree tree = + IndexedTree(store, depth, 1); + + indexed_leaf zero_leaf = { 0, 0, 0 }; + EXPECT_EQ(tree.size(), 1); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf(zero_leaf)); + + // Add 20 random values to the tree + for (size_t i = 0; i < 20; i++) { + auto value = fr::random_element(); + tree.add_value(value); + } + + auto abs_diff = [](uint256_t a, uint256_t b) { + if (a > b) { + return (a - b); + } else { + return (b - a); + } + }; + + // Check if a new random value is not a member of this tree. + fr new_member = fr::random_element(); + std::vector differences; + for (size_t i = 0; i < size_t(tree.size()); i++) { + uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); + uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); + differences.push_back(diff_hi + diff_lo); + } + auto it = std::min_element(differences.begin(), differences.end()); + auto index = static_cast(it - differences.begin()); + + // Merkle proof at `index` proves non-membership of `new_member` + auto hash_path = tree.get_hash_path(index); + EXPECT_TRUE(check_hash_path(tree.root(), hash_path, tree.get_leaf(index), index)); +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.cpp new file mode 100644 index 000000000000..cd29811455dc --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.cpp @@ -0,0 +1,49 @@ +#include "leaves_cache.hpp" + +namespace bb::crypto::merkle_tree { + +index_t LeavesCache::get_size() const +{ + return index_t(leaves_.size()); +} + +std::pair LeavesCache::find_low_value(const fr& new_value) const +{ + std::map::const_iterator it = indices_.lower_bound(new_value); + if (it == indices_.end()) { + // there is no element >= the requested value. + // decrement the iterator to get the value preceeding the requested value + --it; + return std::make_pair(false, it->second); + } + if (it->first == uint256_t(new_value)) { + // the value is already present and the iterator points to it + return std::make_pair(true, it->second); + } + // the iterator points to the element immediately larger than the requested value + --it; + // it now points to the value less than that requested + return std::make_pair(false, it->second); +} +indexed_leaf LeavesCache::get_leaf(const index_t& index) const +{ + ASSERT(index >= 0 && index < leaves_.size()); + return leaves_[size_t(index)]; +} +void LeavesCache::set_at_index(const index_t& index, const indexed_leaf& leaf, bool add_to_index) +{ + if (index >= leaves_.size()) { + leaves_.resize(size_t(index + 1)); + } + leaves_[size_t(index)] = leaf; + if (add_to_index) { + indices_[uint256_t(leaf.value)] = index; + } +} +void LeavesCache::append_leaf(const indexed_leaf& leaf) +{ + index_t next_index = leaves_.size(); + set_at_index(next_index, leaf, true); +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp new file mode 100644 index 000000000000..b38f556807ea --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp @@ -0,0 +1,27 @@ +#pragma once +#include "barretenberg/stdlib/primitives/field/field.hpp" +#include "indexed_leaf.hpp" + +namespace bb::crypto::merkle_tree { + +typedef uint256_t index_t; + +/** + * @brief Used to facilitate testing of the IndexedTree. Stores leaves in memory with an index for O(logN) retrieval of + * 'low leaves' + * + */ +class LeavesCache { + public: + index_t get_size() const; + std::pair find_low_value(const bb::fr& new_value) const; + indexed_leaf get_leaf(const index_t& index) const; + void set_at_index(const index_t& index, const indexed_leaf& leaf, bool add_to_index); + void append_leaf(const indexed_leaf& leaf); + + private: + std::map indices_; + std::vector leaves_; +}; + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.hpp similarity index 92% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.hpp index ddb743b6e8ec..4d38e4311c8e 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.hpp @@ -1,14 +1,16 @@ #pragma once +#include "barretenberg/crypto/merkle_tree/memory_store.hpp" +#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp" #include "barretenberg/stdlib/hash/pedersen/pedersen.hpp" -#include "barretenberg/stdlib/merkle_tree/memory_store.hpp" -#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp" #include "barretenberg/stdlib/primitives/byte_array/byte_array.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" #include "hash_path.hpp" -namespace bb::stdlib::merkle_tree { +namespace bb::crypto::merkle_tree { -template using bit_vector = std::vector>; +template using bit_vector = std::vector>; +template using field_t = bb::stdlib::field_t; +template using bool_t = bb::stdlib::bool_t; /** * Computes the new merkle root if the subtree is correctly inserted at a specified index in a Merkle tree. * @@ -42,9 +44,9 @@ field_t compute_subtree_root(hash_path const& hashes, field_t left = field_t::conditional_assign(path_bit, hashes[i].first, current); field_t right = field_t::conditional_assign(path_bit, current, hashes[i].second); if (is_updating_tree) { - current = pedersen_hash::hash({ left, right }, 0); + current = bb::stdlib::pedersen_hash::hash({ left, right }, 0); } else { - current = pedersen_hash::hash_skip_field_validation({ left, right }, 0); + current = bb::stdlib::pedersen_hash::hash_skip_field_validation({ left, right }, 0); } } @@ -143,7 +145,7 @@ void assert_check_membership(field_t const& root, bool const is_updating_tree = false, std::string const& msg = "assert_check_membership") { - auto exists = stdlib::merkle_tree::check_membership(root, hashes, value, index, is_updating_tree); + auto exists = check_membership(root, hashes, value, index, is_updating_tree); exists.assert_equal(true, msg); } @@ -255,7 +257,7 @@ template field_t compute_tree_root(std::vector 1) { std::vector> next_layer(layer.size() / 2); for (size_t i = 0; i < next_layer.size(); ++i) { - next_layer[i] = pedersen_hash::hash({ layer[i * 2], layer[i * 2 + 1] }); + next_layer[i] = bb::stdlib::pedersen_hash::hash({ layer[i * 2], layer[i * 2 + 1] }); } layer = std::move(next_layer); } @@ -316,7 +318,7 @@ void batch_update_membership(field_t const& new_root, new_root, rollup_root, old_root, old_path, zero_subtree_root, start_index.decompose_into_bits(), height, msg); } -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree namespace bb::stdlib { /** @@ -331,8 +333,8 @@ template static void generate_merkle_membership_test_circuit( using namespace stdlib; using field_ct = field_t; using witness_ct = witness_t; - using MemStore = merkle_tree::MemoryStore; - using MerkleTree_ct = merkle_tree::MerkleTree; + using MemStore = crypto::merkle_tree::MemoryStore; + using MerkleTree_ct = crypto::merkle_tree::MerkleTree; MemStore store; const size_t tree_depth = 7; @@ -348,8 +350,11 @@ template static void generate_merkle_membership_test_circuit( auto idx_ct = field_ct(witness_ct(&builder, fr(idx))).decompose_into_bits(); auto value_ct = field_ct(value); - merkle_tree::check_membership( - root_ct, merkle_tree::create_witness_hash_path(builder, merkle_tree.get_hash_path(idx)), value_ct, idx_ct); + crypto::merkle_tree::check_membership( + root_ct, + crypto::merkle_tree::create_witness_hash_path(builder, merkle_tree.get_hash_path(idx)), + value_ct, + idx_ct); } } } // namespace bb::stdlib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.test.cpp similarity index 89% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.test.cpp index 1d8d06820e74..3975c4abccd7 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/membership.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/membership.test.cpp @@ -8,7 +8,7 @@ #include "barretenberg/proof_system/circuit_builder/ultra_circuit_builder.hpp" using namespace bb; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; using namespace bb::stdlib; namespace { @@ -17,14 +17,15 @@ auto& engine = numeric::get_debug_randomness(); using Builder = UltraCircuitBuilder; -using bool_ct = bool_t; -using field_ct = field_t; -using witness_ct = witness_t; +using bool_ct = bb::stdlib::bool_t; +using field_ct = bb::stdlib::field_t; +using witness_ct = bb::stdlib::witness_t; +using tree = MerkleTree; -TEST(stdlib_merkle_tree, test_check_membership) +TEST(crypto_merkle_tree, test_check_membership) { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = tree(store, 3); auto builder = Builder(); // Check membership at index 0. @@ -48,10 +49,10 @@ TEST(stdlib_merkle_tree, test_check_membership) EXPECT_EQ(result, true); } -TEST(stdlib_merkle_tree, test_batch_update_membership) +TEST(crypto_merkle_tree, test_batch_update_membership) { MemoryStore store; - MerkleTree db(store, 4); + tree db(store, 4); auto builder = Builder(); // Fill in an arbitrary value at i = 2. db.update_element(2, fr::random_element()); @@ -74,10 +75,10 @@ TEST(stdlib_merkle_tree, test_batch_update_membership) EXPECT_EQ(result, true); } -TEST(stdlib_merkle_tree, test_assert_check_membership) +TEST(crypto_merkle_tree, test_assert_check_membership) { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = tree(store, 3); auto builder = Builder(); auto zero = field_ct(witness_ct(&builder, fr::zero())).decompose_into_bits(); @@ -91,10 +92,10 @@ TEST(stdlib_merkle_tree, test_assert_check_membership) EXPECT_EQ(result, true); } -TEST(stdlib_merkle_tree, test_assert_check_membership_fail) +TEST(crypto_merkle_tree, test_assert_check_membership_fail) { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = tree(store, 3); auto builder = Builder(); @@ -109,11 +110,11 @@ TEST(stdlib_merkle_tree, test_assert_check_membership_fail) EXPECT_EQ(result, false); } // To test whether both old hash path and new hash path works for the same Merkle tree -TEST(stdlib_merkle_tree, test_update_members) +TEST(crypto_merkle_tree, test_update_members) { { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = tree(store, 3); auto builder = Builder(); @@ -137,7 +138,7 @@ TEST(stdlib_merkle_tree, test_update_members) } { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = tree(store, 3); auto builder = Builder(); @@ -161,13 +162,13 @@ TEST(stdlib_merkle_tree, test_update_members) } } -TEST(stdlib_merkle_tree, test_tree) +TEST(crypto_merkle_tree, test_tree) { size_t depth = 3; size_t num = 1UL << depth; MemoryStore store; - MerkleTree db(store, depth); - MemoryTree mem_tree(depth); + tree db(store, depth); + MemoryTree mem_tree(depth); auto builder = Builder(); @@ -183,11 +184,11 @@ TEST(stdlib_merkle_tree, test_tree) EXPECT_EQ(result, true); } -TEST(stdlib_merkle_tree, test_update_memberships) +TEST(crypto_merkle_tree, test_update_memberships) { constexpr size_t depth = 4; MemoryStore store; - MerkleTree tree(store, depth); + tree tree(store, depth); auto builder = Builder(); diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_store.hpp similarity index 96% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_store.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_store.hpp index ee1f4196a740..a1b5dfec6691 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_store.hpp @@ -4,7 +4,7 @@ #include #include -namespace bb::stdlib::merkle_tree { +namespace bb::crypto::merkle_tree { class MemoryStore { public: @@ -84,4 +84,4 @@ class MemoryStore { std::set deletes_; }; -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.hpp new file mode 100644 index 000000000000..e85e613aa9dc --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.hpp @@ -0,0 +1,114 @@ +#pragma once +#include "hash_path.hpp" + +namespace bb::crypto::merkle_tree { + +/** + * A MemoryTree is structured as follows: + * hashes_ + * +------------------------------------------------------------------------------+ + * | 0 -> h_{0,0} h_{0,1} h_{0,2} h_{0,3} h_{0,4} h_{0,5} h_{0,6} h_{0,7} | + * i | | + * n | 8 -> h_{1,0} h_{1,1} h_{1,2} h_{1,3} | + * d | | + * e | 12 -> h_{2,0} h_{2,1} | + * x | | + * | 14 -> h_{3,0} | + * +------------------------------------------------------------------------------+ + * + * Here, depth_ = 3 and {h_{0,j}}_{i=0..7} are leaf values. + * Also, root_ = h_{3,0} and total_size_ = (2 * 8 - 2) = 14. + * Lastly, h_{i,j} = hash( h_{i-1,2j}, h_{i-1,2j+1} ) where i > 1. + */ +template class MemoryTree { + public: + MemoryTree(size_t depth); + + fr_hash_path get_hash_path(size_t index); + + fr_sibling_path get_sibling_path(size_t index); + + fr update_element(size_t index, fr const& value); + + fr root() const { return root_; } + + public: + size_t depth_; + size_t total_size_; + bb::fr root_; + std::vector hashes_; +}; + +template +MemoryTree::MemoryTree(size_t depth) + : depth_(depth) +{ + + ASSERT(depth_ >= 1 && depth <= 20); + total_size_ = 1UL << depth_; + hashes_.resize(total_size_ * 2 - 2); + + // Build the entire tree. + auto current = fr(0); + size_t layer_size = total_size_; + for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { + for (size_t i = 0; i < layer_size; ++i) { + hashes_[offset + i] = current; + } + current = HashingPolicy::hash_pair(current, current); + } + + root_ = current; +} + +template fr_hash_path MemoryTree::get_hash_path(size_t index) +{ + fr_hash_path path(depth_); + size_t offset = 0; + size_t layer_size = total_size_; + for (size_t i = 0; i < depth_; ++i) { + index -= index & 0x1; + path[i] = std::make_pair(hashes_[offset + index], hashes_[offset + index + 1]); + offset += layer_size; + layer_size >>= 1; + index >>= 1; + } + return path; +} + +template fr_sibling_path MemoryTree::get_sibling_path(size_t index) +{ + fr_sibling_path path(depth_); + size_t offset = 0; + size_t layer_size = total_size_; + for (size_t i = 0; i < depth_; i++) { + if (index % 2 == 0) { + path[i] = hashes_[offset + index + 1]; + } else { + path[i] = hashes_[offset + index - 1]; + } + offset += layer_size; + layer_size >>= 1; + index >>= 1; + } + return path; +} + +template fr MemoryTree::update_element(size_t index, fr const& value) +{ + size_t offset = 0; + size_t layer_size = total_size_; + fr current = value; + for (size_t i = 0; i < depth_; ++i) { + hashes_[offset + index] = current; + index &= (~0ULL) - 1; + current = HashingPolicy::hash_pair(hashes_[offset + index], hashes_[offset + index + 1]); + offset += layer_size; + layer_size >>= 1; + index >>= 1; + } + root_ = current; + return root_; +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.test.cpp similarity index 74% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.test.cpp index cfbf5e8a6a90..56cca4b03bc9 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/memory_tree.test.cpp @@ -2,7 +2,9 @@ #include using namespace bb; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; + +using HashPolicy = PedersenHashPolicy; static std::vector VALUES = []() { std::vector values(4); @@ -12,17 +14,17 @@ static std::vector VALUES = []() { return values; }(); -TEST(stdlib_merkle_tree, test_memory_store) +TEST(crypto_merkle_tree, test_memory_store) { fr e00 = 0; fr e01 = VALUES[1]; fr e02 = VALUES[2]; fr e03 = VALUES[3]; - fr e10 = hash_pair_native(e00, e01); - fr e11 = hash_pair_native(e02, e03); - fr root = hash_pair_native(e10, e11); + fr e10 = HashPolicy::hash_pair(e00, e01); + fr e11 = HashPolicy::hash_pair(e02, e03); + fr root = HashPolicy::hash_pair(e10, e11); - MemoryTree db(2); + MemoryTree db(2); for (size_t i = 0; i < 4; ++i) { db.update_element(i, VALUES[i]); } @@ -43,17 +45,17 @@ TEST(stdlib_merkle_tree, test_memory_store) EXPECT_EQ(db.root(), root); } -TEST(stdlib_merkle_tree, test_memory_store_sibling_path) +TEST(crypto_merkle_tree, test_memory_store_sibling_path) { fr e00 = 0; fr e01 = VALUES[1]; fr e02 = VALUES[2]; fr e03 = VALUES[3]; - fr e10 = hash_pair_native(e00, e01); - fr e11 = hash_pair_native(e02, e03); - fr root = hash_pair_native(e10, e11); + fr e10 = HashPolicy::hash_pair(e00, e01); + fr e11 = HashPolicy::hash_pair(e02, e03); + fr root = HashPolicy::hash_pair(e10, e11); - MemoryTree db(2); + MemoryTree db(2); for (size_t i = 0; i < 4; ++i) { db.update_element(i, VALUES[i]); } diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.hpp similarity index 67% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.hpp index 6ea45c94e756..576f5bfcced4 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.hpp @@ -1,16 +1,104 @@ -#include "merkle_tree.hpp" +#pragma once #include "barretenberg/common/net.hpp" #include "barretenberg/numeric/bitop/count_leading_zeros.hpp" #include "barretenberg/numeric/bitop/keep_n_lsb.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" -#include "hash.hpp" -#include "memory_store.hpp" +#include "barretenberg/stdlib/primitives/field/field.hpp" +#include "hash_path.hpp" +#include "merkle_tree.hpp" #include #include -namespace bb::stdlib::merkle_tree { - -using namespace bb; +namespace bb::crypto::merkle_tree { + +class MemoryStore; + +template class MerkleTree { + public: + typedef uint256_t index_t; + + MerkleTree(Store& store, size_t depth, uint8_t tree_id = 0); + MerkleTree(MerkleTree const& other) = delete; + MerkleTree(MerkleTree&& other); + ~MerkleTree(); + + fr_hash_path get_hash_path(index_t index); + + fr_sibling_path get_sibling_path(index_t index); + + fr update_element(index_t index, fr const& value); + + fr root() const; + + size_t depth() const { return depth_; } + + index_t size() const; + + protected: + void load_metadata(); + + /** + * Computes the root hash of a tree of `height`, that is empty other than `value` at `index`. + * + * @param height: The tree depth + * @param index: the index of the non-empty leaf + * @param value: the value to be stored in the non-empty leaf + * + * @see Check full documentation: https://hackmd.io/2zyJc6QhRuugyH8D78Tbqg?view + */ + fr update_element(fr const& root, fr const& value, index_t index, size_t height); + + fr get_element(fr const& root, index_t index, size_t height); + + /** + * Computes the root hash of a tree of `height`, that is empty other than `value` at `index`. + * + * @param height: The tree depth + * @param index: the index of the non-empty leaf + * @param value: the value to be stored in the non-empty leaf + */ + fr compute_zero_path_hash(size_t height, index_t index, fr const& value); + + /** + * Given child nodes `a` and `b` and index of `a`, compute their parent node `p` and store [p : (a, b)]. + * + * @param a_index: the index of the child node `a` + * @param a: child node + * @param b: child node + * @param height: the height of the parent node + */ + fr binary_put(index_t a_index, fr const& a, fr const& b, size_t height); + + fr fork_stump( + fr const& value1, index_t index1, fr const& value2, index_t index2, size_t height, size_t stump_height); + + /** + * Stores a parent node and child nodes in the database as [key : (left, right)]. + * + * @param key: The node value to be stored as key + * @param left: the left child node + * @param right: the right child node + */ + void put(fr const& key, fr const& left, fr const& right); + + /** + * Stores a stump [key : (value, index, true)] in the memory. + * The additional byte `true` is to denote this is a stump. + * + * @param key: The node value to be stored as key + * @param value: value of the non-empty leaf in the stump + * @param index: the index of the non-empty leaf in the stump + */ + void put_stump(fr const& key, index_t index, fr const& value); + + void remove(fr const& key); + + protected: + Store& store_; + std::vector zero_hashes_; + size_t depth_; + uint8_t tree_id_; +}; // Size of merkle tree nodes in bytes. constexpr size_t REGULAR_NODE_SIZE = 64; @@ -21,8 +109,8 @@ template inline bool bit_set(T const& index, size_t i) return bool((index >> i) & 0x1); } -template -MerkleTree::MerkleTree(Store& store, size_t depth, uint8_t tree_id) +template +MerkleTree::MerkleTree(Store& store, size_t depth, uint8_t tree_id) : store_(store) , depth_(depth) , tree_id_(tree_id) @@ -34,29 +122,30 @@ MerkleTree::MerkleTree(Store& store, size_t depth, uint8_t tree_id) auto current = fr(0); for (size_t i = 0; i < depth; ++i) { zero_hashes_[i] = current; - current = hash_pair_native(current, current); + current = HashingPolicy::hash_pair(current, current); } } -template -MerkleTree::MerkleTree(MerkleTree&& other) +template +MerkleTree::MerkleTree(MerkleTree&& other) : store_(other.store_) , zero_hashes_(std::move(other.zero_hashes_)) , depth_(other.depth_) , tree_id_(other.tree_id_) {} -template MerkleTree::~MerkleTree() {} +template MerkleTree::~MerkleTree() {} -template fr MerkleTree::root() const +template fr MerkleTree::root() const { std::vector root; std::vector key = { tree_id_ }; bool status = store_.get(key, root); - return status ? from_buffer(root) : hash_pair_native(zero_hashes_.back(), zero_hashes_.back()); + return status ? from_buffer(root) : HashingPolicy::hash_pair(zero_hashes_.back(), zero_hashes_.back()); } -template typename MerkleTree::index_t MerkleTree::size() const +template +typename MerkleTree::index_t MerkleTree::size() const { std::vector size_buf; std::vector key = { tree_id_ }; @@ -64,7 +153,8 @@ template typename MerkleTree::index_t MerkleTree: return status ? from_buffer(size_buf, 32) : 0; } -template fr_hash_path MerkleTree::get_hash_path(index_t index) +template +fr_hash_path MerkleTree::get_hash_path(index_t index) { fr_hash_path path(depth_); @@ -105,7 +195,7 @@ template fr_hash_path MerkleTree::get_hash_path(index_t } else { path[j] = std::make_pair(current, zero_hashes_[j]); } - current = hash_pair_native(path[j].first, path[j].second); + current = HashingPolicy::hash_pair(path[j].first, path[j].second); } } else { // Requesting path to a different, independent element. @@ -125,7 +215,7 @@ template fr_hash_path MerkleTree::get_hash_path(index_t } else { path[j] = std::make_pair(current, zero_hashes_[j]); } - current = hash_pair_native(path[j].first, path[j].second); + current = HashingPolicy::hash_pair(path[j].first, path[j].second); } } break; @@ -135,7 +225,8 @@ template fr_hash_path MerkleTree::get_hash_path(index_t return path; } -template fr_sibling_path MerkleTree::get_sibling_path(index_t index) +template +fr_sibling_path MerkleTree::get_sibling_path(index_t index) { fr_sibling_path path(depth_); @@ -194,7 +285,8 @@ template fr_sibling_path MerkleTree::get_sibling_path(in return path; } -template fr MerkleTree::update_element(index_t index, fr const& value) +template +fr MerkleTree::update_element(index_t index, fr const& value) { auto leaf = value; using serialize::write; @@ -214,18 +306,19 @@ template fr MerkleTree::update_element(index_t index, fr return r; } -template fr MerkleTree::binary_put(index_t a_index, fr const& a, fr const& b, size_t height) +template +fr MerkleTree::binary_put(index_t a_index, fr const& a, fr const& b, size_t height) { bool a_is_right = bit_set(a_index, height - 1); auto left = a_is_right ? b : a; auto right = a_is_right ? a : b; - auto key = hash_pair_native(left, right); + auto key = HashingPolicy::hash_pair(left, right); put(key, left, right); return key; } -template -fr MerkleTree::fork_stump( +template +fr MerkleTree::fork_stump( fr const& value1, index_t index1, fr const& value2, index_t index2, size_t height, size_t common_height) { if (height == common_height) { @@ -250,8 +343,8 @@ fr MerkleTree::fork_stump( } } -template -fr MerkleTree::update_element(fr const& root, fr const& value, index_t index, size_t height) +template +fr MerkleTree::update_element(fr const& root, fr const& value, index_t index, size_t height) { // Base layer of recursion at height = 0. if (height == 0) { @@ -297,7 +390,7 @@ fr MerkleTree::update_element(fr const& root, fr const& value, index_t in } else { left = subtree_root; } - auto new_root = hash_pair_native(left, right); + auto new_root = HashingPolicy::hash_pair(left, right); put(new_root, left, right); // Remove the old node only while rolling back in recursion. @@ -308,7 +401,8 @@ fr MerkleTree::update_element(fr const& root, fr const& value, index_t in } } -template fr MerkleTree::compute_zero_path_hash(size_t height, index_t index, fr const& value) +template +fr MerkleTree::compute_zero_path_hash(size_t height, index_t index, fr const& value) { fr current = value; for (size_t i = 0; i < height; ++i) { @@ -321,12 +415,13 @@ template fr MerkleTree::compute_zero_path_hash(size_t he right = zero_hashes_[i]; left = current; } - current = hash_pair_native(left, right); + current = HashingPolicy::hash_pair(left, right); } return current; } -template void MerkleTree::put(fr const& key, fr const& left, fr const& right) +template +void MerkleTree::put(fr const& key, fr const& left, fr const& right) { std::vector value; write(value, left); @@ -334,7 +429,8 @@ template void MerkleTree::put(fr const& key, fr const& l store_.put(key.to_buffer(), value); } -template void MerkleTree::put_stump(fr const& key, index_t index, fr const& value) +template +void MerkleTree::put_stump(fr const& key, index_t index, fr const& value) { std::vector buf; write(buf, value); @@ -344,11 +440,9 @@ template void MerkleTree::put_stump(fr const& key, index store_.put(key.to_buffer(), buf); } -template void MerkleTree::remove(fr const& key) +template void MerkleTree::remove(fr const& key) { store_.del(key.to_buffer()); } -template class MerkleTree; - -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.test.cpp similarity index 78% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.test.cpp index 55bd530c2161..085f97555abe 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/merkle_tree.test.cpp @@ -6,7 +6,7 @@ #include "memory_tree.hpp" using namespace bb::stdlib; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; using Builder = UltraCircuitBuilder; @@ -25,13 +25,13 @@ static std::vector VALUES = []() { return values; }(); -TEST(stdlib_merkle_tree, test_kv_memory_vs_memory_consistency) +TEST(crypto_merkle_tree, test_kv_memory_vs_memory_consistency) { constexpr size_t depth = 10; - MemoryTree memdb(depth); + MemoryTree memdb(depth); MemoryStore store; - MerkleTree db(store, depth); + MerkleTree db(store, depth); std::vector indicies(1 << depth); std::iota(indicies.begin(), indicies.end(), 0); @@ -53,10 +53,10 @@ TEST(stdlib_merkle_tree, test_kv_memory_vs_memory_consistency) EXPECT_EQ(db.root(), memdb.root()); } -TEST(stdlib_merkle_tree, test_size) +TEST(crypto_merkle_tree, test_size) { MemoryStore store; - auto db = MerkleTree(store, 256); + auto db = MerkleTree(store, 256); EXPECT_EQ(db.size(), 0ULL); @@ -81,12 +81,12 @@ TEST(stdlib_merkle_tree, test_size) EXPECT_EQ(db.size(), 3ULL); } -TEST(stdlib_merkle_tree, test_get_hash_path) +TEST(crypto_merkle_tree, test_get_hash_path) { - MemoryTree memdb(10); + MemoryTree memdb(10); MemoryStore store; - auto db = MerkleTree(store, 10); + auto db = MerkleTree(store, 10); EXPECT_EQ(memdb.get_hash_path(512), db.get_hash_path(512)); @@ -103,12 +103,12 @@ TEST(stdlib_merkle_tree, test_get_hash_path) EXPECT_EQ(db.get_hash_path(512), memdb.get_hash_path(512)); } -TEST(stdlib_merkle_tree, test_get_sibling_path) +TEST(crypto_merkle_tree, test_get_sibling_path) { - MemoryTree memdb(10); + MemoryTree memdb(10); MemoryStore store; - auto db = MerkleTree(store, 10); + auto db = MerkleTree(store, 10); EXPECT_EQ(memdb.get_sibling_path(512), db.get_sibling_path(512)); @@ -125,11 +125,11 @@ TEST(stdlib_merkle_tree, test_get_sibling_path) EXPECT_EQ(db.get_sibling_path(512), memdb.get_sibling_path(512)); } -TEST(stdlib_merkle_tree, test_get_hash_path_layers) +TEST(crypto_merkle_tree, test_get_hash_path_layers) { { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = MerkleTree(store, 3); auto before = db.get_hash_path(1); db.update_element(0, VALUES[1]); @@ -142,7 +142,7 @@ TEST(stdlib_merkle_tree, test_get_hash_path_layers) { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = MerkleTree(store, 3); auto before = db.get_hash_path(7); db.update_element(0x0, VALUES[1]); @@ -154,11 +154,11 @@ TEST(stdlib_merkle_tree, test_get_hash_path_layers) } } -TEST(stdlib_merkle_tree, test_get_sibling_path_layers) +TEST(crypto_merkle_tree, test_get_sibling_path_layers) { { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = MerkleTree(store, 3); auto before = db.get_sibling_path(1); db.update_element(0, VALUES[1]); @@ -171,7 +171,7 @@ TEST(stdlib_merkle_tree, test_get_sibling_path_layers) { MemoryStore store; - auto db = MerkleTree(store, 3); + auto db = MerkleTree(store, 3); auto before = db.get_sibling_path(7); db.update_element(0x0, VALUES[1]); diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp similarity index 76% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp index d1bd3d310d87..1ac402ac3102 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_leaf.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp @@ -1,10 +1,10 @@ #pragma once #include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" #include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/stdlib/hash/poseidon2/poseidon2.hpp" -namespace bb::stdlib::merkle_tree { +namespace bb::crypto::merkle_tree { -using namespace bb; typedef uint256_t index_t; struct nullifier_leaf { @@ -22,14 +22,16 @@ struct nullifier_leaf { return os; } - bb::fr hash() const { return stdlib::merkle_tree::hash_native({ value, nextIndex, nextValue }); } + std::vector get_hash_inputs() const { return std::vector{ value, nextIndex, nextValue }; } + + static nullifier_leaf zero() { return nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }; } }; /** * @brief Wrapper for the Nullifier leaf class that allows for 0 values * */ -class WrappedNullifierLeaf { +template class WrappedNullifierLeaf { public: // Initialize with a nullifier leaf @@ -70,21 +72,26 @@ class WrappedNullifierLeaf { * * @return bb::fr */ - bb::fr hash() const { return data.has_value() ? data.value().hash() : bb::fr::zero(); } + bb::fr hash() const + { + return data.has_value() ? HashingPolicy::hash(data.value().get_hash_inputs()) : bb::fr::zero(); + } /** * @brief Generate a zero leaf (call the constructor with no arguments) * * @return NullifierLeaf */ - static WrappedNullifierLeaf zero() { return WrappedNullifierLeaf(); } + static WrappedNullifierLeaf zero() { return WrappedNullifierLeaf(); } private: // Underlying data std::optional data; }; -inline std::pair find_closest_leaf(std::vector const& leaves_, fr const& new_value) +template +inline std::pair find_closest_leaf(std::vector> const& leaves_, + fr const& new_value) { std::vector diff; bool repeated = false; @@ -111,4 +118,4 @@ inline std::pair find_closest_leaf(std::vector(it - diff.begin()), repeated); } -} // namespace bb::stdlib::merkle_tree +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp new file mode 100644 index 000000000000..5b9fdb94ee2d --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp @@ -0,0 +1,174 @@ +#pragma once +#include "../hash.hpp" +#include "../memory_tree.hpp" +#include "nullifier_leaf.hpp" + +namespace bb::crypto::merkle_tree { + +/** + * An NullifierMemoryTree is structured just like a usual merkle tree: + * + * hashes_ + * +------------------------------------------------------------------------------+ + * | 0 -> h_{0,0} h_{0,1} h_{0,2} h_{0,3} h_{0,4} h_{0,5} h_{0,6} h_{0,7} | + * i | | + * n | 8 -> h_{1,0} h_{1,1} h_{1,2} h_{1,3} | + * d | | + * e | 12 -> h_{2,0} h_{2,1} | + * x | | + * | 14 -> h_{3,0} | + * +------------------------------------------------------------------------------+ + * + * Here, depth_ = 3 and {h_{0,j}}_{i=0..7} are leaf values. + * Also, root_ = h_{3,0} and total_size_ = (2 * 8 - 2) = 14. + * Lastly, h_{i,j} = hash( h_{i-1,2j}, h_{i-1,2j+1} ) where i > 1. + * + * 1. Initial state: + * + * # + * + * # # + * + * # # # # + * + * # # # # # # # # + * + * index 0 1 2 3 4 5 6 7 + * + * val 0 0 0 0 0 0 0 0 + * nextIdx 0 0 0 0 0 0 0 0 + * nextVal 0 0 0 0 0 0 0 0 + * + * 2. Add new leaf with value 30 + * + * val 0 30 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 30 0 0 0 0 0 0 0 + * + * 3. Add new leaf with value 10 + * + * val 0 30 10 0 0 0 0 0 + * nextIdx 2 0 1 0 0 0 0 0 + * nextVal 10 0 30 0 0 0 0 0 + * + * 4. Add new leaf with value 20 + * + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + * + * 5. Add new leaf with value 50 + * + * val 0 30 10 20 50 0 0 0 + * nextIdx 2 4 3 1 0 0 0 0 + * nextVal 10 50 20 30 0 0 0 0 + */ +template class NullifierMemoryTree : public MemoryTree { + + public: + NullifierMemoryTree(size_t depth, size_t initial_size = 1); + + using MemoryTree::get_hash_path; + using MemoryTree::root; + using MemoryTree::update_element; + + fr_hash_path update_element(fr const& value); + + const std::vector& get_hashes() { return hashes_; } + const WrappedNullifierLeaf get_leaf(size_t index) + { + return (index < leaves_.size()) ? leaves_[index] : WrappedNullifierLeaf(nullifier_leaf::zero()); + } + const std::vector>& get_leaves() { return leaves_; } + + protected: + using MemoryTree::depth_; + using MemoryTree::hashes_; + using MemoryTree::root_; + using MemoryTree::total_size_; + std::vector> leaves_; +}; + +template +NullifierMemoryTree::NullifierMemoryTree(size_t depth, size_t initial_size) + : MemoryTree(depth) +{ + ASSERT(depth_ >= 1 && depth <= 32); + ASSERT(initial_size > 0); + total_size_ = 1UL << depth_; + hashes_.resize(total_size_ * 2 - 2); + + // Build the entire tree and fill with 0 hashes. + auto current = WrappedNullifierLeaf(nullifier_leaf::zero()).hash(); + size_t layer_size = total_size_; + for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { + for (size_t i = 0; i < layer_size; ++i) { + hashes_[offset + i] = current; + } + current = HashingPolicy::hash_pair(current, current); + } + + // Insert the initial leaves + for (size_t i = 0; i < initial_size; i++) { + auto initial_leaf = + WrappedNullifierLeaf(nullifier_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }); + leaves_.push_back(initial_leaf); + } + + leaves_[initial_size - 1] = WrappedNullifierLeaf( + nullifier_leaf{ .value = leaves_[initial_size - 1].unwrap().value, .nextIndex = 0, .nextValue = 0 }); + + for (size_t i = 0; i < initial_size; ++i) { + update_element(i, leaves_[i].hash()); + } +} + +template fr_hash_path NullifierMemoryTree::update_element(fr const& value) +{ + // Find the leaf with the value closest and less than `value` + + // If value is 0 we simply append 0 a null NullifierLeaf to the tree + fr_hash_path hash_path; + if (value == 0) { + auto zero_leaf = WrappedNullifierLeaf::zero(); + hash_path = get_hash_path(leaves_.size() - 1); + leaves_.push_back(zero_leaf); + update_element(leaves_.size() - 1, zero_leaf.hash()); + return hash_path; + } + + size_t current; + bool is_already_present; + std::tie(current, is_already_present) = find_closest_leaf(leaves_, value); + + nullifier_leaf current_leaf = leaves_[current].unwrap(); + nullifier_leaf new_leaf = { .value = value, + .nextIndex = current_leaf.nextIndex, + .nextValue = current_leaf.nextValue }; + + if (!is_already_present) { + // Update the current leaf to point it to the new leaf + current_leaf.nextIndex = leaves_.size(); + current_leaf.nextValue = value; + + leaves_[current].set(current_leaf); + + // Insert the new leaf with (nextIndex, nextValue) of the current leaf + leaves_.push_back(new_leaf); + } + + hash_path = get_hash_path(current); + // Update the old leaf in the tree + auto old_leaf_hash = HashingPolicy::hash(current_leaf.get_hash_inputs()); + size_t old_leaf_index = current; + auto root = update_element(old_leaf_index, old_leaf_hash); + + // Insert the new leaf in the tree + auto new_leaf_hash = HashingPolicy::hash(new_leaf.get_hash_inputs()); + size_t new_leaf_index = is_already_present ? old_leaf_index : leaves_.size() - 1; + root = update_element(new_leaf_index, new_leaf_hash); + + return hash_path; +} + +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp similarity index 67% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp index eecce8b87af7..e33051822047 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp @@ -2,7 +2,10 @@ #include using namespace bb; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; + +using HashPolicy = PedersenHashPolicy; +using WrappedLeaf = WrappedNullifierLeaf; void print_tree(const size_t depth, std::vector hashes, std::string const& msg) { @@ -20,7 +23,7 @@ void print_tree(const size_t depth, std::vector hashes, std::string const& m bool check_hash_path(const fr& root, const fr_hash_path& path, const nullifier_leaf& leaf_value, const size_t idx) { - auto current = leaf_value.hash(); + auto current = WrappedLeaf(leaf_value).hash(); size_t depth_ = path.size(); size_t index = idx; for (size_t i = 0; i < depth_; ++i) { @@ -36,7 +39,7 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) { // Create a depth-3 indexed merkle tree constexpr size_t depth = 3; - NullifierMemoryTree tree(depth); + NullifierMemoryTree tree(depth); /** * Intial state: @@ -49,7 +52,7 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ nullifier_leaf zero_leaf = { 0, 0, 0 }; EXPECT_EQ(tree.get_leaves().size(), 1); - EXPECT_EQ(tree.get_leaves()[0], zero_leaf); + EXPECT_EQ(tree.get_leaves()[0].unwrap(), zero_leaf); /** * Add new value 30: @@ -62,8 +65,8 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(30); EXPECT_EQ(tree.get_leaves().size(), 2); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); /** * Add new value 10: @@ -76,9 +79,9 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(10); EXPECT_EQ(tree.get_leaves().size(), 3); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 1, 30 }).hash()); /** * Add new value 20: @@ -91,18 +94,18 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); // Adding the same value must not affect anything tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); /** * Add new value 50: @@ -115,11 +118,11 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) */ tree.update_element(50); EXPECT_EQ(tree.get_leaves().size(), 5); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 4, 50 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf({ 50, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 4, 50 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf({ 50, 0, 0 }).hash()); // Manually compute the node values auto e000 = tree.get_leaves()[0].hash(); @@ -127,18 +130,18 @@ TEST(crypto_nullifier_tree, test_nullifier_memory) auto e010 = tree.get_leaves()[2].hash(); auto e011 = tree.get_leaves()[3].hash(); auto e100 = tree.get_leaves()[4].hash(); - auto e101 = WrappedNullifierLeaf::zero().hash(); - auto e110 = WrappedNullifierLeaf::zero().hash(); - auto e111 = WrappedNullifierLeaf::zero().hash(); + auto e101 = WrappedLeaf(zero_leaf).hash(); + auto e110 = WrappedLeaf(zero_leaf).hash(); + auto e111 = WrappedLeaf(zero_leaf).hash(); - auto e00 = hash_pair_native(e000, e001); - auto e01 = hash_pair_native(e010, e011); - auto e10 = hash_pair_native(e100, e101); - auto e11 = hash_pair_native(e110, e111); + auto e00 = HashPolicy::hash_pair(e000, e001); + auto e01 = HashPolicy::hash_pair(e010, e011); + auto e10 = HashPolicy::hash_pair(e100, e101); + auto e11 = HashPolicy::hash_pair(e110, e111); - auto e0 = hash_pair_native(e00, e01); - auto e1 = hash_pair_native(e10, e11); - auto root = hash_pair_native(e0, e1); + auto e0 = HashPolicy::hash_pair(e00, e01); + auto e1 = HashPolicy::hash_pair(e10, e11); + auto root = HashPolicy::hash_pair(e0, e1); // Check the hash path at index 2 and 3 // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) @@ -165,7 +168,7 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) { // Create a depth-3 indexed merkle tree constexpr size_t depth = 3; - NullifierMemoryTree tree(depth); + NullifierMemoryTree tree(depth); /** * Intial state: @@ -176,7 +179,7 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) * nextIdx 0 0 0 0 0 0 0 0 * nextVal 0 0 0 0 0 0 0 0 */ - WrappedNullifierLeaf zero_leaf = WrappedNullifierLeaf({ 0, 0, 0 }); + WrappedLeaf zero_leaf = WrappedLeaf({ 0, 0, 0 }); EXPECT_EQ(tree.get_leaves().size(), 1); EXPECT_EQ(tree.get_leaves()[0], zero_leaf); @@ -191,8 +194,8 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(30); EXPECT_EQ(tree.get_leaves().size(), 2); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); /** * Add new value 10: @@ -205,9 +208,9 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(10); EXPECT_EQ(tree.get_leaves().size(), 3); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 1, 30 }).hash()); /** * Add new value 20: @@ -220,18 +223,18 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); // Adding the same value must not affect anything tree.update_element(20); EXPECT_EQ(tree.get_leaves().size(), 4); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); /** * Add new value 0: @@ -244,11 +247,11 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(0); EXPECT_EQ(tree.get_leaves().size(), 5); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf::zero().hash()); /* * Add new value 0: @@ -261,12 +264,12 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(0); EXPECT_EQ(tree.get_leaves().size(), 6); - EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedNullifierLeaf({ 30, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedNullifierLeaf::zero().hash()); - EXPECT_EQ(tree.get_leaves()[5].hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaves()[1].hash(), WrappedLeaf({ 30, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaves()[2].hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaves()[3].hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaves()[4].hash(), WrappedLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaves()[5].hash(), WrappedLeaf::zero().hash()); /** * Add new value 50: @@ -279,14 +282,14 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) */ tree.update_element(50); EXPECT_EQ(tree.get_leaves().size(), 7); - EXPECT_EQ(tree.get_leaf(0).hash(), WrappedNullifierLeaf({ 0, 2, 10 }).hash()); - EXPECT_EQ(tree.get_leaf(1).hash(), WrappedNullifierLeaf({ 30, 6, 50 }).hash()); - EXPECT_EQ(tree.get_leaf(2).hash(), WrappedNullifierLeaf({ 10, 3, 20 }).hash()); - EXPECT_EQ(tree.get_leaf(3).hash(), WrappedNullifierLeaf({ 20, 1, 30 }).hash()); - EXPECT_EQ(tree.get_leaf(4).hash(), WrappedNullifierLeaf::zero().hash()); - EXPECT_EQ(tree.get_leaf(5).hash(), WrappedNullifierLeaf::zero().hash()); - EXPECT_EQ(tree.get_leaf(6).hash(), WrappedNullifierLeaf({ 50, 0, 0 }).hash()); - EXPECT_EQ(tree.get_leaf(7).hash(), WrappedNullifierLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaf(0).hash(), WrappedLeaf({ 0, 2, 10 }).hash()); + EXPECT_EQ(tree.get_leaf(1).hash(), WrappedLeaf({ 30, 6, 50 }).hash()); + EXPECT_EQ(tree.get_leaf(2).hash(), WrappedLeaf({ 10, 3, 20 }).hash()); + EXPECT_EQ(tree.get_leaf(3).hash(), WrappedLeaf({ 20, 1, 30 }).hash()); + EXPECT_EQ(tree.get_leaf(4).hash(), WrappedLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaf(5).hash(), WrappedLeaf::zero().hash()); + EXPECT_EQ(tree.get_leaf(6).hash(), WrappedLeaf({ 50, 0, 0 }).hash()); + EXPECT_EQ(tree.get_leaf(7).hash(), zero_leaf.hash()); // Manually compute the node values auto e000 = tree.get_leaf(0).hash(); @@ -298,14 +301,22 @@ TEST(crypto_nullifier_tree, test_nullifier_memory_appending_zero) auto e110 = tree.get_leaf(6).hash(); auto e111 = tree.get_leaf(7).hash(); - auto e00 = hash_pair_native(e000, e001); - auto e01 = hash_pair_native(e010, e011); - auto e10 = hash_pair_native(e100, e101); - auto e11 = hash_pair_native(e110, e111); + auto e00 = HashPolicy::hash_pair(e000, e001); + auto e01 = HashPolicy::hash_pair(e010, e011); + auto e10 = HashPolicy::hash_pair(e100, e101); + auto e11 = HashPolicy::hash_pair(e110, e111); + + auto e0 = HashPolicy::hash_pair(e00, e01); + auto e1 = HashPolicy::hash_pair(e10, e11); + auto root = HashPolicy::hash_pair(e0, e1); - auto e0 = hash_pair_native(e00, e01); - auto e1 = hash_pair_native(e10, e11); - auto root = hash_pair_native(e0, e1); + fr_hash_path expected1 = { + std::make_pair(e000, e001), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(0), expected1); + EXPECT_EQ(tree.get_hash_path(1), expected1); // Check the hash path at index 2 and 3 // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) @@ -331,11 +342,11 @@ TEST(crypto_nullifier_tree, test_nullifier_tree) { // Create a depth-8 indexed merkle tree constexpr size_t depth = 8; - NullifierMemoryTree tree(depth); + NullifierMemoryTree tree(depth); nullifier_leaf zero_leaf = { 0, 0, 0 }; EXPECT_EQ(tree.get_leaves().size(), 1); - EXPECT_EQ(tree.get_leaves()[0].hash(), zero_leaf.hash()); + EXPECT_EQ(tree.get_leaves()[0].hash(), WrappedLeaf(zero_leaf).hash()); // Add 20 random values to the tree for (size_t i = 0; i < 20; i++) { diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp similarity index 51% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp index 502ea05c1b13..95ee0ea72abb 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.cpp @@ -9,43 +9,48 @@ #include #include -namespace bb::stdlib::merkle_tree { +namespace bb::crypto::merkle_tree { -template inline bool bit_set(T const& index, size_t i) -{ - return bool((index >> i) & 0x1); -} - -template -NullifierTree::NullifierTree(Store& store, size_t depth, uint8_t tree_id) - : MerkleTree(store, depth, tree_id) +template +NullifierTree::NullifierTree(Store& store, size_t depth, size_t initial_size, uint8_t tree_id) + : MerkleTree(store, depth, tree_id) { ASSERT(depth_ >= 1 && depth <= 256); + ASSERT(initial_size > 0); zero_hashes_.resize(depth); - // Compute the zero values at each layer. - // Insert the zero leaf to the `leaves` and also to the tree at index 0. - WrappedNullifierLeaf initial_leaf = - WrappedNullifierLeaf(nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }); - leaves.push_back(initial_leaf); - update_element(0, initial_leaf.hash()); - // Create the zero hashes for the tree - auto current = WrappedNullifierLeaf::zero().hash(); + auto current = + WrappedNullifierLeaf(nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }).hash(); for (size_t i = 0; i < depth; ++i) { zero_hashes_[i] = current; - current = hash_pair_native(current, current); + current = HashingPolicy::hash_pair(current, current); + } + + // Insert the initial leaves + for (size_t i = 0; i < initial_size; i++) { + auto initial_leaf = + WrappedNullifierLeaf(nullifier_leaf{ .value = i, .nextIndex = i + 1, .nextValue = i + 1 }); + leaves.push_back(initial_leaf); + } + + leaves[initial_size - 1] = WrappedNullifierLeaf( + nullifier_leaf{ .value = leaves[initial_size - 1].unwrap().value, .nextIndex = 0, .nextValue = 0 }); + + for (size_t i = 0; i < initial_size; ++i) { + update_element(i, leaves[i].hash()); } } -template -NullifierTree::NullifierTree(NullifierTree&& other) - : MerkleTree(std::move(other)) +template +NullifierTree::NullifierTree(NullifierTree&& other) + : MerkleTree(std::move(other)) {} -template NullifierTree::~NullifierTree() {} +template NullifierTree::~NullifierTree() {} -template fr NullifierTree::update_element(fr const& value) +template +fr NullifierTree::update_element(fr const& value) { // Find the leaf with the value closest and less than `value` size_t current; @@ -53,7 +58,7 @@ template fr NullifierTree::update_element(fr const& valu std::tie(current, is_already_present) = find_closest_leaf(leaves, value); nullifier_leaf current_leaf = leaves[current].unwrap(); - WrappedNullifierLeaf new_leaf = WrappedNullifierLeaf( + WrappedNullifierLeaf new_leaf = WrappedNullifierLeaf( { .value = value, .nextIndex = current_leaf.nextIndex, .nextValue = current_leaf.nextValue }); if (!is_already_present) { // Update the current leaf to point it to the new leaf @@ -79,6 +84,6 @@ template fr NullifierTree::update_element(fr const& valu return r; } -template class NullifierTree; +template class NullifierTree; -} // namespace bb::stdlib::merkle_tree \ No newline at end of file +} // namespace bb::crypto::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.hpp new file mode 100644 index 000000000000..ea727de6855f --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.hpp @@ -0,0 +1,37 @@ +#pragma once +#include "../hash.hpp" +#include "../merkle_tree.hpp" +#include "nullifier_leaf.hpp" + +namespace bb::crypto::merkle_tree { + +template class NullifierTree : public MerkleTree { + public: + typedef uint256_t index_t; + + NullifierTree(Store& store, size_t depth, size_t initial_size = 1, uint8_t tree_id = 0); + NullifierTree(NullifierTree const& other) = delete; + NullifierTree(NullifierTree&& other); + ~NullifierTree(); + + using MerkleTree::get_hash_path; + using MerkleTree::root; + using MerkleTree::size; + using MerkleTree::depth; + + fr update_element(fr const& value); + + private: + using MerkleTree::update_element; + using MerkleTree::get_element; + using MerkleTree::compute_zero_path_hash; + + private: + using MerkleTree::store_; + using MerkleTree::zero_hashes_; + using MerkleTree::depth_; + using MerkleTree::tree_id_; + std::vector> leaves; +}; + +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.test.cpp similarity index 81% rename from barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.test.cpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.test.cpp index 99a1eeeffdeb..41ddc1670b29 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_tree.test.cpp @@ -6,17 +6,18 @@ #include "nullifier_memory_tree.hpp" using namespace bb; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; namespace { auto& engine = numeric::get_debug_randomness(); auto& random_engine = numeric::get_randomness(); } // namespace +const size_t NUM_VALUES = 1024; static std::vector VALUES = []() { - std::vector values(1024); - for (size_t i = 0; i < 1024; ++i) { - values[i] = fr(random_engine.get_random_uint64()); + std::vector values(NUM_VALUES); + for (size_t i = 0; i < NUM_VALUES; ++i) { + values[i] = fr(random_engine.get_random_uint256()); } return values; }(); @@ -38,10 +39,12 @@ inline void print_tree(const size_t depth, std::vector hashes, std::string c TEST(stdlib_nullifier_tree, test_kv_memory_vs_memory_consistency) { constexpr size_t depth = 2; - NullifierMemoryTree memdb(depth); + NullifierMemoryTree memdb(depth); MemoryStore store; - NullifierTree db(store, depth); + NullifierTree db(store, depth); + + EXPECT_EQ(db.root(), memdb.root()); std::vector indicies(1 << depth); std::iota(indicies.begin(), indicies.end(), 0); @@ -66,7 +69,7 @@ TEST(stdlib_nullifier_tree, test_kv_memory_vs_memory_consistency) TEST(stdlib_nullifier_tree, test_size) { MemoryStore store; - auto db = NullifierTree(store, 256); + auto db = NullifierTree(store, 256); // We assume that the first leaf is already filled with (0, 0, 0). EXPECT_EQ(db.size(), 1ULL); @@ -90,10 +93,10 @@ TEST(stdlib_nullifier_tree, test_size) TEST(stdlib_nullifier_tree, test_get_hash_path) { - NullifierMemoryTree memdb(10); + NullifierMemoryTree memdb(10); MemoryStore store; - auto db = NullifierTree(store, 10); + auto db = NullifierTree(store, 10); EXPECT_EQ(memdb.get_hash_path(512), db.get_hash_path(512)); @@ -114,7 +117,7 @@ TEST(stdlib_nullifier_tree, test_get_hash_path_layers) { { MemoryStore store; - auto db = NullifierTree(store, 3); + auto db = NullifierTree(store, 3); auto before = db.get_hash_path(1); db.update_element(VALUES[1]); @@ -127,7 +130,7 @@ TEST(stdlib_nullifier_tree, test_get_hash_path_layers) { MemoryStore store; - auto db = NullifierTree(store, 3); + auto db = NullifierTree(store, 3); auto before = db.get_hash_path(7); db.update_element(VALUES[1]); diff --git a/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp b/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp index 86f505105baf..b455ab5d6d50 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.cpp @@ -5,23 +5,40 @@ extern "C" { -WASM_EXPORT void pedersen_hash(uint8_t const* inputs_buffer, uint32_t const* hash_index, uint8_t* output) +WASM_EXPORT void pedersen_hash(bb::fr::vec_in_buf inputs_buffer, uint32_t const* hash_index, bb::fr::out_buf output) { std::vector to_hash; read(inputs_buffer, to_hash); - crypto::GeneratorContext ctx; + bb::crypto::GeneratorContext ctx; ctx.offset = static_cast(ntohl(*hash_index)); - auto r = crypto::pedersen_hash::hash(to_hash, ctx); + auto r = bb::crypto::pedersen_hash::hash(to_hash, ctx); bb::fr::serialize_to_buffer(r, output); } -WASM_EXPORT void pedersen_hash_buffer(uint8_t const* input_buffer, uint32_t const* hash_index, uint8_t* output) +WASM_EXPORT void pedersen_hashes(bb::fr::vec_in_buf inputs_buffer, uint32_t const* hash_index, bb::fr::out_buf output) +{ + std::vector to_hash; + read(inputs_buffer, to_hash); + bb::crypto::GeneratorContext ctx; + ctx.offset = static_cast(ntohl(*hash_index)); + const size_t numHashes = to_hash.size() / 2; + std::vector results; + size_t count = 0; + while (count < numHashes) { + auto r = bb::crypto::pedersen_hash::hash({ to_hash[count * 2], to_hash[count * 2 + 1] }, ctx); + results.push_back(r); + ++count; + } + write(output, results); +} + +WASM_EXPORT void pedersen_hash_buffer(uint8_t const* input_buffer, uint32_t const* hash_index, bb::fr::out_buf output) { std::vector to_hash; read(input_buffer, to_hash); - crypto::GeneratorContext ctx; + bb::crypto::GeneratorContext ctx; ctx.offset = static_cast(ntohl(*hash_index)); - auto r = crypto::pedersen_hash::hash_buffer(to_hash, ctx); + auto r = bb::crypto::pedersen_hash::hash_buffer(to_hash, ctx); bb::fr::serialize_to_buffer(r, output); } } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp b/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp index b3d8d50c30dc..ecd51260201e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp @@ -5,9 +5,7 @@ extern "C" { -using namespace bb; - -WASM_EXPORT void pedersen_hash(fr::vec_in_buf inputs_buffer, uint32_t const* hash_index, fr::out_buf output); - -WASM_EXPORT void pedersen_hash_buffer(uint8_t const* input_buffer, uint32_t const* hash_index, fr::out_buf output); +WASM_EXPORT void pedersen_hash(bb::fr::vec_in_buf inputs_buffer, uint32_t const* hash_index, bb::fr::out_buf output); +WASM_EXPORT void pedersen_hashes(bb::fr::vec_in_buf inputs_buffer, uint32_t const* hash_index, bb::fr::out_buf output); +WASM_EXPORT void pedersen_hash_buffer(uint8_t const* input_buffer, uint32_t const* hash_index, bb::fr::out_buf output); } \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.cpp b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.cpp new file mode 100644 index 000000000000..bd43c64915b2 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.cpp @@ -0,0 +1,32 @@ +#include "c_bind.hpp" +#include "barretenberg/common/mem.hpp" +#include "barretenberg/common/serialize.hpp" +#include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp" +#include "poseidon2.hpp" + +extern "C" { + +WASM_EXPORT void poseidon_hash(bb::fr::vec_in_buf inputs_buffer, bb::fr::out_buf output) +{ + std::vector to_hash; + read(inputs_buffer, to_hash); + auto r = bb::crypto::Poseidon2::hash(to_hash); + bb::fr::serialize_to_buffer(r, output); +} + +WASM_EXPORT void poseidon_hashes(bb::fr::vec_in_buf inputs_buffer, bb::fr::out_buf output) +{ + std::vector to_hash; + read(inputs_buffer, to_hash); + const size_t numHashes = to_hash.size() / 2; + std::vector results; + size_t count = 0; + while (count < numHashes) { + auto r = bb::crypto::Poseidon2::hash( + { to_hash[count * 2], to_hash[count * 2 + 1] }); + results.push_back(r); + ++count; + } + write(output, results); +} +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp new file mode 100644 index 000000000000..c7dc58555485 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include "barretenberg/common/wasm_export.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" + +extern "C" { + +WASM_EXPORT void poseidon_hash(bb::fr::vec_in_buf inputs_buffer, bb::fr::out_buf output); +WASM_EXPORT void poseidon_hashes(bb::fr::vec_in_buf inputs_buffer, bb::fr::out_buf output); +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt index d9a4c241e42f..14399cc02b22 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt @@ -6,7 +6,7 @@ barretenberg_module( stdlib_blake2s stdlib_keccak stdlib_pedersen_hash - stdlib_merkle_tree + crypto_merkle_tree stdlib_schnorr crypto_sha256 ) diff --git a/barretenberg/cpp/src/barretenberg/dsl/types.hpp b/barretenberg/cpp/src/barretenberg/dsl/types.hpp index b5bf296cbf53..8dd4a55ac9ff 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/types.hpp +++ b/barretenberg/cpp/src/barretenberg/dsl/types.hpp @@ -1,11 +1,11 @@ #pragma once #include "barretenberg/plonk/composer/ultra_composer.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/goblin/goblin.hpp" #include "barretenberg/plonk/proof_system/prover/prover.hpp" #include "barretenberg/stdlib/commitment/pedersen/pedersen.hpp" #include "barretenberg/stdlib/encryption/schnorr/schnorr.hpp" -#include "barretenberg/stdlib/merkle_tree/hash_path.hpp" #include "barretenberg/stdlib/primitives/bigfield/bigfield.hpp" #include "barretenberg/stdlib/primitives/biggroup/biggroup.hpp" #include "barretenberg/stdlib/primitives/bit_array/bit_array.hpp" @@ -58,7 +58,7 @@ using bn254 = bb::stdlib::bn254; using secp256k1_ct = bb::stdlib::secp256k1; using secp256r1_ct = bb::stdlib::secp256r1; -using hash_path_ct = bb::stdlib::merkle_tree::hash_path; +using hash_path_ct = bb::crypto::merkle_tree::hash_path; using schnorr_signature_bits_ct = bb::stdlib::schnorr_signature_bits; diff --git a/barretenberg/cpp/src/barretenberg/goblin/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/goblin/CMakeLists.txt index ee4b04affe82..60ba6aa607c3 100644 --- a/barretenberg/cpp/src/barretenberg/goblin/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/goblin/CMakeLists.txt @@ -1 +1 @@ -barretenberg_module(goblin stdlib_recursion ultra_honk eccvm translator_vm stdlib_sha256 stdlib_merkle_tree stdlib_primitives) \ No newline at end of file +barretenberg_module(goblin stdlib_recursion ultra_honk eccvm translator_vm stdlib_sha256 crypto_merkle_tree stdlib_primitives) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp b/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp index 12fde5fb31b6..74efb83c615d 100644 --- a/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp +++ b/barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp @@ -2,15 +2,15 @@ #include "barretenberg/commitment_schemes/commitment_key.hpp" #include "barretenberg/crypto/ecdsa/ecdsa.hpp" +#include "barretenberg/crypto/merkle_tree/membership.hpp" +#include "barretenberg/crypto/merkle_tree/memory_store.hpp" +#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp" #include "barretenberg/flavor/goblin_ultra.hpp" #include "barretenberg/goblin/goblin.hpp" #include "barretenberg/proof_system/circuit_builder/goblin_ultra_circuit_builder.hpp" #include "barretenberg/srs/global_crs.hpp" #include "barretenberg/stdlib/encryption/ecdsa/ecdsa.hpp" #include "barretenberg/stdlib/hash/sha256/sha256.hpp" -#include "barretenberg/stdlib/merkle_tree/membership.hpp" -#include "barretenberg/stdlib/merkle_tree/memory_store.hpp" -#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp" #include "barretenberg/stdlib/primitives/curves/secp256k1.hpp" #include "barretenberg/stdlib/primitives/packed_byte_array/packed_byte_array.hpp" #include "barretenberg/stdlib/recursion/honk/verifier/protogalaxy_recursive_verifier.hpp" diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/CMakeLists.txt index 2a44f93e256e..9c90ad2ba844 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/CMakeLists.txt @@ -8,4 +8,4 @@ barretenberg_module( stdlib_pedersen_commitment stdlib_schnorr stdlib_primitives - stdlib_merkle_tree) \ No newline at end of file + crypto_merkle_tree) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/compute_circuit_data.cpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/compute_circuit_data.cpp index da7f5037d003..7d8533dd600f 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/compute_circuit_data.cpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/compute_circuit_data.cpp @@ -1,7 +1,7 @@ #include "compute_circuit_data.hpp" #include "../notes/native/index.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/join_split_example/types.hpp" -#include "barretenberg/stdlib/merkle_tree/hash_path.hpp" #include "join_split_circuit.hpp" #include "sign_join_split_tx.hpp" @@ -10,7 +10,7 @@ namespace bb::join_split_example::proofs::join_split { using namespace bb::join_split_example::proofs::join_split; using namespace bb::stdlib; using namespace bb::join_split_example::proofs::notes::native; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; join_split_tx noop_tx() { diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.cpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.cpp index 5eae28bf8de1..73cae41dbee9 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.cpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.cpp @@ -7,7 +7,7 @@ namespace bb::join_split_example::proofs::join_split { using namespace bb::plonk; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; static std::shared_ptr proving_key; static std::shared_ptr verification_key; diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp index 5d110351dbb8..aa530c87a276 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split.test.cpp @@ -5,15 +5,15 @@ #include "../notes/native/index.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" +#include "barretenberg/crypto/merkle_tree/index.hpp" #include "barretenberg/join_split_example/types.hpp" #include "barretenberg/plonk/proof_system/proving_key/serialize.hpp" -#include "barretenberg/stdlib/merkle_tree/index.hpp" #include "index.hpp" #include "join_split_circuit.hpp" namespace bb::join_split_example::proofs::join_split { -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; /* Old join-split tests below. The value of having all of these logic tests is unclear, but we'll leave them around, at least for a while. */ @@ -27,7 +27,7 @@ constexpr bool CIRCUIT_CHANGE_EXPECTED = false; using namespace bb; using namespace bb::stdlib; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; using namespace bb::join_split_example::proofs::notes::native; using key_pair = join_split_example::fixtures::grumpkin_key_pair; @@ -53,7 +53,7 @@ class join_split_tests : public ::testing::Test { virtual void SetUp() { store = std::make_unique(); - tree = std::make_unique>(*store, 32); + tree = std::make_unique>(*store, 32); user = join_split_example::fixtures::create_user_context(); default_value_note = { .value = 100, @@ -302,7 +302,7 @@ class join_split_tests : public ::testing::Test { join_split_example::fixtures::user_context user; std::unique_ptr store; - std::unique_ptr> tree; + std::unique_ptr> tree; bridge_call_data empty_bridge_call_data = { .bridge_address_id = 0, .input_asset_id_a = 0, .input_asset_id_b = 0, diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_circuit.cpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_circuit.cpp index 847e7d15cff7..109deccf8294 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_circuit.cpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_circuit.cpp @@ -4,15 +4,15 @@ #include "../notes/circuit/claim/claim_note.hpp" #include "../notes/circuit/value/compute_nullifier.hpp" #include "../notes/circuit/value/value_note.hpp" +#include "barretenberg/crypto/merkle_tree/membership.hpp" #include "barretenberg/join_split_example/types.hpp" -#include "barretenberg/stdlib/merkle_tree/membership.hpp" #include "verify_signature.hpp" namespace bb::join_split_example::proofs::join_split { using namespace bb::plonk; using namespace notes::circuit; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; using namespace bb::crypto; /** @@ -31,7 +31,7 @@ field_ct process_input_note(field_ct const& account_private_key, const bool_ct valid_value = note.value == 0 || is_note_in_use; valid_value.assert_equal(true, "padding note non zero"); - const bool_ct exists = bb::stdlib::merkle_tree::check_membership( + const bool_ct exists = bb::crypto::merkle_tree::check_membership( merkle_root, hash_path, note.commitment, index.value.decompose_into_bits(DATA_TREE_DEPTH)); const bool_ct valid = exists || is_propagated || !is_note_in_use; valid.assert_equal(true, "input note not a member"); @@ -217,7 +217,7 @@ join_split_outputs join_split_circuit_component(join_split_inputs const& inputs) const auto account_alias_hash = inputs.alias_hash; const auto account_note_data = account::account_note(account_alias_hash.value, account_public_key, signer); const bool_ct signing_key_exists = - stdlib::merkle_tree::check_membership(inputs.merkle_root, + crypto::merkle_tree::check_membership(inputs.merkle_root, inputs.account_note_path, account_note_data.commitment, inputs.account_note_index.value.decompose_into_bits(DATA_TREE_DEPTH)); @@ -288,8 +288,8 @@ void join_split_circuit(Builder& builder, join_split_tx const& tx) .signing_pub_key = group_ct::from_witness(&builder, tx.signing_pub_key), .signature = stdlib::schnorr_convert_signature(&builder, tx.signature), .merkle_root = witness_ct(&builder, tx.old_data_root), - .input_path1 = stdlib::merkle_tree::create_witness_hash_path(builder, tx.input_path[0]), - .input_path2 = stdlib::merkle_tree::create_witness_hash_path(builder, tx.input_path[1]), + .input_path1 = crypto::merkle_tree::create_witness_hash_path(builder, tx.input_path[0]), + .input_path2 = crypto::merkle_tree::create_witness_hash_path(builder, tx.input_path[1]), .account_note_index = suint_ct(witness_ct(&builder, tx.account_note_index), DATA_TREE_DEPTH, "account_note_index"), .account_note_path = merkle_tree::create_witness_hash_path(builder, tx.account_note_path), diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_js_parity.test.cpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_js_parity.test.cpp index 6754c48cbd0d..ca3cbf3e2cb9 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_js_parity.test.cpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_js_parity.test.cpp @@ -3,16 +3,16 @@ #include "../notes/native/index.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" +#include "barretenberg/crypto/merkle_tree/index.hpp" #include "barretenberg/crypto/sha256/sha256.hpp" #include "barretenberg/plonk/proof_system/proving_key/serialize.hpp" -#include "barretenberg/stdlib/merkle_tree/index.hpp" #include "index.hpp" namespace bb::join_split_example::proofs::join_split { using namespace bb; // using namespace bb::stdlib::types; -using namespace bb::stdlib::merkle_tree; +using namespace bb::crypto::merkle_tree; using namespace bb::join_split_example::proofs::notes::native; using key_pair = join_split_example::fixtures::grumpkin_key_pair; @@ -32,7 +32,7 @@ class join_split_js_parity_tests : public ::testing::Test { virtual void SetUp() { store = std::make_unique(); - tree = std::make_unique>(*store, 32); + tree = std::make_unique>(*store, 32); } void append_notes(std::vector const& notes) @@ -56,7 +56,7 @@ class join_split_js_parity_tests : public ::testing::Test { } std::unique_ptr store; - std::unique_ptr> tree; + std::unique_ptr> tree; }; TEST_F(join_split_js_parity_tests, test_full_proof) diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_tx.hpp b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_tx.hpp index eb1282f00065..1e164be26877 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_tx.hpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/join_split/join_split_tx.hpp @@ -1,9 +1,9 @@ #pragma once #include "../notes/native/claim/claim_note_tx_data.hpp" #include "../notes/native/value/value_note.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/crypto/schnorr/schnorr.hpp" #include "barretenberg/join_split_example/types.hpp" -#include "barretenberg/stdlib/merkle_tree/hash_path.hpp" namespace bb::join_split_example::proofs::join_split { @@ -15,7 +15,7 @@ struct join_split_tx { uint32_t num_input_notes; std::array input_index; bb::fr old_data_root; - std::array input_path; + std::array input_path; std::array input_note; std::array output_note; @@ -25,7 +25,7 @@ struct join_split_tx { bb::fr alias_hash; bool account_required; uint32_t account_note_index; - bb::stdlib::merkle_tree::fr_hash_path account_note_path; + bb::crypto::merkle_tree::fr_hash_path account_note_path; grumpkin::g1::affine_element signing_pub_key; bb::fr backward_link; // 0: no link, otherwise: any commitment. diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/notes/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/notes/CMakeLists.txt index 0efb8cfc2750..ad23167bf652 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/proofs/notes/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/join_split_example/proofs/notes/CMakeLists.txt @@ -6,4 +6,4 @@ barretenberg_module( stdlib_pedersen_commitment stdlib_schnorr stdlib_primitives - stdlib_merkle_tree) \ No newline at end of file + crypto_merkle_tree) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/join_split_example/types.hpp b/barretenberg/cpp/src/barretenberg/join_split_example/types.hpp index 68b0e79363d7..fbc2e45f7ca3 100644 --- a/barretenberg/cpp/src/barretenberg/join_split_example/types.hpp +++ b/barretenberg/cpp/src/barretenberg/join_split_example/types.hpp @@ -4,10 +4,10 @@ #include "barretenberg/plonk/composer/ultra_composer.hpp" #include "barretenberg/ultra_honk/ultra_composer.hpp" +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/plonk/proof_system/prover/prover.hpp" #include "barretenberg/stdlib/commitment/pedersen/pedersen.hpp" #include "barretenberg/stdlib/encryption/schnorr/schnorr.hpp" -#include "barretenberg/stdlib/merkle_tree/hash_path.hpp" #include "barretenberg/stdlib/primitives/bool/bool.hpp" #include "barretenberg/stdlib/primitives/byte_array/byte_array.hpp" #include "barretenberg/stdlib/primitives/curves/bn254.hpp" @@ -37,7 +37,7 @@ using pedersen_commitment = bb::stdlib::pedersen_commitment; using pedersen_hash = bb::stdlib::pedersen_hash; using bn254 = bb::stdlib::bn254; -using hash_path_ct = bb::stdlib::merkle_tree::hash_path; +using hash_path_ct = bb::crypto::merkle_tree::hash_path; using schnorr_signature_bits = bb::stdlib::schnorr_signature_bits; diff --git a/barretenberg/cpp/src/barretenberg/stdlib/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/stdlib/CMakeLists.txt index c3fa01c8518c..dea2efa6ea36 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/stdlib/CMakeLists.txt @@ -2,5 +2,4 @@ add_subdirectory(hash) add_subdirectory(commitment) add_subdirectory(encryption) add_subdirectory(primitives) -add_subdirectory(recursion) -add_subdirectory(merkle_tree) \ No newline at end of file +add_subdirectory(recursion) \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/hash/pedersen/pedersen.bench.cpp b/barretenberg/cpp/src/barretenberg/stdlib/hash/pedersen/pedersen.bench.cpp index d0658eef7095..6258d855e940 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/hash/pedersen/pedersen.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/hash/pedersen/pedersen.bench.cpp @@ -104,6 +104,18 @@ void native_pedersen_eight_hash_bench(State& state) noexcept } BENCHMARK(native_pedersen_eight_hash_bench)->MinTime(3); +void native_pedersen_hash_pair_bench(State& state) noexcept +{ + std::vector elements(2); + for (size_t i = 0; i < 2; ++i) { + elements[i] = grumpkin::fq::random_element(); + } + for (auto _ : state) { + crypto::pedersen_hash::hash(elements); + } +} +BENCHMARK(native_pedersen_hash_pair_bench)->Unit(benchmark::kMillisecond)->MinTime(3); + void construct_pedersen_witnesses_bench(State& state) noexcept { bb::srs::init_crs_factory(BARRETENBERG_SRS_PATH); diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/CMakeLists.txt deleted file mode 100644 index 0553ea7c3a62..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -barretenberg_module(stdlib_merkle_tree stdlib_primitives stdlib_blake3s stdlib_pedersen_hash) diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/indexed_tree/indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/indexed_tree/indexed_tree.test.cpp new file mode 100644 index 000000000000..d1dc13d95e52 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/indexed_tree/indexed_tree.test.cpp @@ -0,0 +1,331 @@ +#include "indexed_tree.hpp" +#include "../array_store.hpp" +#include "../hash.hpp" +#include "../nullifier_tree/nullifier_memory_tree.hpp" +#include "barretenberg/common/streams.hpp" +#include "barretenberg/common/test.hpp" +#include "barretenberg/numeric/random/engine.hpp" +#include "leaves_cache.hpp" + +using namespace bb; +using namespace bb::crypto::merkle_tree; + +using HashPolicy = Poseidon2HashPolicy; + +namespace { +auto& engine = numeric::get_debug_randomness(); +auto& random_engine = numeric::get_randomness(); +} // namespace + +const size_t NUM_VALUES = 1024; +static std::vector VALUES = []() { + std::vector values(NUM_VALUES); + for (size_t i = 0; i < NUM_VALUES; ++i) { + values[i] = fr(random_engine.get_random_uint256()); + } + return values; +}(); + +TEST(stdlib_indexed_tree, can_create) +{ + ArrayStore store(10); + IndexedTree tree = IndexedTree(store, 10); + EXPECT_EQ(tree.size(), 1ULL); + + NullifierMemoryTree memdb(10); + EXPECT_EQ(memdb.root(), tree.root()); +} + +TEST(stdlib_indexed_tree, test_size) +{ + ArrayStore store(32); + auto db = IndexedTree(store, 32); + + // We assume that the first leaf is already filled with (0, 0, 0). + EXPECT_EQ(db.size(), 1ULL); + + // Add a new non-zero leaf at index 1. + db.add_value(30); + EXPECT_EQ(db.size(), 2ULL); + + // Add third. + db.add_value(10); + EXPECT_EQ(db.size(), 3ULL); + + // Add forth. + db.add_value(20); + EXPECT_EQ(db.size(), 4ULL); +} + +TEST(stdlib_indexed_tree, test_get_hash_path) +{ + NullifierMemoryTree memdb(10); + + ArrayStore store(10); + auto db = IndexedTree(store, 10); + + EXPECT_EQ(memdb.root(), db.root()); + + EXPECT_EQ(memdb.get_hash_path(0), db.get_hash_path(0)); + + memdb.update_element(VALUES[512]); + db.add_value(VALUES[512]); + + EXPECT_EQ(db.get_hash_path(0), memdb.get_hash_path(0)); + + for (size_t i = 0; i < 512; ++i) { + memdb.update_element(VALUES[i]); + db.add_value(VALUES[i]); + } + + EXPECT_EQ(db.get_hash_path(512), memdb.get_hash_path(512)); +} + +TEST(stdlib_indexed_tree, test_batch_insert) +{ + const size_t batch_size = 16; + const size_t num_batches = 16; + size_t depth = 10; + NullifierMemoryTree memdb(depth, batch_size); + + ArrayStore store1(depth); + IndexedTree tree1 = + IndexedTree(store1, depth, batch_size); + + ArrayStore store2(depth); + IndexedTree tree2 = + IndexedTree(store2, depth, batch_size); + + EXPECT_EQ(memdb.root(), tree1.root()); + EXPECT_EQ(tree1.root(), tree2.root()); + + EXPECT_EQ(memdb.get_hash_path(0), tree1.get_hash_path(0)); + EXPECT_EQ(tree1.get_hash_path(0), tree2.get_hash_path(0)); + + EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); + EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); + + for (size_t i = 0; i < num_batches; i++) { + std::vector batch; + std::vector memory_tree_hash_paths; + for (size_t j = 0; j < batch_size; j++) { + batch.push_back(fr(random_engine.get_random_uint256())); + fr_hash_path path = memdb.update_element(batch[j]); + memory_tree_hash_paths.push_back(path); + } + std::vector tree1_hash_paths = tree1.add_or_update_values(batch, true); + std::vector tree2_hash_paths = tree2.add_or_update_values(batch); + EXPECT_EQ(memdb.root(), tree1.root()); + EXPECT_EQ(tree1.root(), tree2.root()); + + EXPECT_EQ(memdb.get_hash_path(0), tree1.get_hash_path(0)); + EXPECT_EQ(tree1.get_hash_path(0), tree2.get_hash_path(0)); + + EXPECT_EQ(memdb.get_hash_path(512), tree1.get_hash_path(512)); + EXPECT_EQ(tree1.get_hash_path(512), tree2.get_hash_path(512)); + + for (size_t j = 0; j < batch_size; j++) { + EXPECT_EQ(tree1_hash_paths[j], tree2_hash_paths[j]); + // EXPECT_EQ(tree1_hash_paths[j], memory_tree_hash_paths[j]); + } + } +} + +fr hash_leaf(const indexed_leaf& leaf) +{ + return HashPolicy::hash(leaf.get_hash_inputs()); +} + +bool check_hash_path(const fr& root, const fr_hash_path& path, const indexed_leaf& leaf_value, const size_t idx) +{ + auto current = hash_leaf(leaf_value); + size_t depth_ = path.size(); + size_t index = idx; + for (size_t i = 0; i < depth_; ++i) { + fr left = (index & 1) ? path[i].first : current; + fr right = (index & 1) ? current : path[i].second; + current = HashPolicy::hash_pair(left, right); + index >>= 1; + } + return current == root; +} + +TEST(stdlib_indexed_tree, test_indexed_memory) +{ + // Create a depth-3 indexed merkle tree + constexpr size_t depth = 3; + ArrayStore store(depth); + IndexedTree tree = + IndexedTree(store, depth, 1); + + /** + * Intial state: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 0 0 0 0 0 0 0 + * nextIdx 0 0 0 0 0 0 0 0 + * nextVal 0 0 0 0 0 0 0 0 + */ + indexed_leaf zero_leaf = { 0, 0, 0 }; + EXPECT_EQ(tree.size(), 1); + EXPECT_EQ(tree.get_leaf(0), zero_leaf); + + /** + * Add new value 30: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 0 0 0 0 0 0 + * nextIdx 1 0 0 0 0 0 0 0 + * nextVal 30 0 0 0 0 0 0 0 + */ + tree.add_value(30); + EXPECT_EQ(tree.size(), 2); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 1, 30 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + + /** + * Add new value 10: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 0 0 0 0 0 + * nextIdx 2 0 1 0 0 0 0 0 + * nextVal 10 0 30 0 0 0 0 0 + */ + tree.add_value(10); + EXPECT_EQ(tree.size(), 3); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 1, 30 })); + + /** + * Add new value 20: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 0 0 0 0 + * nextIdx 2 0 3 1 0 0 0 0 + * nextVal 10 0 20 30 0 0 0 0 + */ + tree.add_value(20); + EXPECT_EQ(tree.size(), 4); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 0, 0 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 3, 20 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(3)), hash_leaf({ 20, 1, 30 })); + + // Adding the same value must not affect anything + // tree.update_element(20); + // EXPECT_EQ(tree.get_leaves().size(), 4); + // EXPECT_EQ(tree.get_leaves()[0], hash_leaf({ 0, 2, 10 })); + // EXPECT_EQ(tree.get_leaves()[1], hash_leaf({ 30, 0, 0 })); + // EXPECT_EQ(tree.get_leaves()[2], hash_leaf({ 10, 3, 20 })); + // EXPECT_EQ(tree.get_leaves()[3], hash_leaf({ 20, 1, 30 })); + + /** + * Add new value 50: + * + * index 0 1 2 3 4 5 6 7 + * --------------------------------------------------------------------- + * val 0 30 10 20 50 0 0 0 + * nextIdx 2 4 3 1 0 0 0 0 + * nextVal 10 50 20 30 0 0 0 0 + */ + tree.add_value(50); + EXPECT_EQ(tree.size(), 5); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf({ 0, 2, 10 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(1)), hash_leaf({ 30, 4, 50 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(2)), hash_leaf({ 10, 3, 20 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(3)), hash_leaf({ 20, 1, 30 })); + EXPECT_EQ(hash_leaf(tree.get_leaf(4)), hash_leaf({ 50, 0, 0 })); + + // Manually compute the node values + auto e000 = hash_leaf(tree.get_leaf(0)); + auto e001 = hash_leaf(tree.get_leaf(1)); + auto e010 = hash_leaf(tree.get_leaf(2)); + auto e011 = hash_leaf(tree.get_leaf(3)); + auto e100 = hash_leaf(tree.get_leaf(4)); + auto e101 = hash_leaf({ 0, 0, 0 }); + auto e110 = hash_leaf({ 0, 0, 0 }); + auto e111 = hash_leaf({ 0, 0, 0 }); + + auto e00 = HashPolicy::hash_pair(e000, e001); + auto e01 = HashPolicy::hash_pair(e010, e011); + auto e10 = HashPolicy::hash_pair(e100, e101); + auto e11 = HashPolicy::hash_pair(e110, e111); + + auto e0 = HashPolicy::hash_pair(e00, e01); + auto e1 = HashPolicy::hash_pair(e10, e11); + auto root = HashPolicy::hash_pair(e0, e1); + + // Check the hash path at index 2 and 3 + // Note: This merkle proof would also serve as a non-membership proof of values in (10, 20) and (20, 30) + fr_hash_path expected = { + std::make_pair(e000, e001), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(0), expected); + EXPECT_EQ(tree.get_hash_path(1), expected); + fr_hash_path expected2 = { + std::make_pair(e010, e011), + std::make_pair(e00, e01), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(2), expected2); + EXPECT_EQ(tree.get_hash_path(3), expected2); + EXPECT_EQ(tree.root(), root); + + // Check the hash path at index 6 and 7 + expected = { + std::make_pair(e110, e111), + std::make_pair(e10, e11), + std::make_pair(e0, e1), + }; + EXPECT_EQ(tree.get_hash_path(6), expected); + EXPECT_EQ(tree.get_hash_path(7), expected); +} + +TEST(stdlib_indexed_tree, test_indexed_tree) +{ + // Create a depth-8 indexed merkle tree + constexpr size_t depth = 8; + ArrayStore store(depth); + IndexedTree tree = + IndexedTree(store, depth, 1); + + indexed_leaf zero_leaf = { 0, 0, 0 }; + EXPECT_EQ(tree.size(), 1); + EXPECT_EQ(hash_leaf(tree.get_leaf(0)), hash_leaf(zero_leaf)); + + // Add 20 random values to the tree + for (size_t i = 0; i < 20; i++) { + auto value = fr::random_element(); + tree.add_value(value); + } + + auto abs_diff = [](uint256_t a, uint256_t b) { + if (a > b) { + return (a - b); + } else { + return (b - a); + } + }; + + // Check if a new random value is not a member of this tree. + fr new_member = fr::random_element(); + std::vector differences; + for (size_t i = 0; i < size_t(tree.size()); i++) { + uint256_t diff_hi = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); + uint256_t diff_lo = abs_diff(uint256_t(new_member), uint256_t(tree.get_leaf(i).value)); + differences.push_back(diff_hi + diff_lo); + } + auto it = std::min_element(differences.begin(), differences.end()); + auto index = static_cast(it - differences.begin()); + + // Merkle proof at `index` proves non-membership of `new_member` + auto hash_path = tree.get_hash_path(index); + EXPECT_TRUE(check_hash_path(tree.root(), hash_path, tree.get_leaf(index), index)); +} \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.cpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.cpp deleted file mode 100644 index d15cd5976acc..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.cpp +++ /dev/null @@ -1,77 +0,0 @@ -#include "memory_tree.hpp" -#include "hash.hpp" - -namespace bb::stdlib::merkle_tree { - -MemoryTree::MemoryTree(size_t depth) - : depth_(depth) -{ - - ASSERT(depth_ >= 1 && depth <= 20); - total_size_ = 1UL << depth_; - hashes_.resize(total_size_ * 2 - 2); - - // Build the entire tree. - auto current = fr(0); - size_t layer_size = total_size_; - for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { - for (size_t i = 0; i < layer_size; ++i) { - hashes_[offset + i] = current; - } - current = hash_pair_native(current, current); - } - - root_ = current; -} - -fr_hash_path MemoryTree::get_hash_path(size_t index) -{ - fr_hash_path path(depth_); - size_t offset = 0; - size_t layer_size = total_size_; - for (size_t i = 0; i < depth_; ++i) { - index -= index & 0x1; - path[i] = std::make_pair(hashes_[offset + index], hashes_[offset + index + 1]); - offset += layer_size; - layer_size >>= 1; - index >>= 1; - } - return path; -} - -fr_sibling_path MemoryTree::get_sibling_path(size_t index) -{ - fr_sibling_path path(depth_); - size_t offset = 0; - size_t layer_size = total_size_; - for (size_t i = 0; i < depth_; i++) { - if (index % 2 == 0) { - path[i] = hashes_[offset + index + 1]; - } else { - path[i] = hashes_[offset + index - 1]; - } - offset += layer_size; - layer_size >>= 1; - index >>= 1; - } - return path; -} - -fr MemoryTree::update_element(size_t index, fr const& value) -{ - size_t offset = 0; - size_t layer_size = total_size_; - fr current = value; - for (size_t i = 0; i < depth_; ++i) { - hashes_[offset + index] = current; - index &= (~0ULL) - 1; - current = hash_pair_native(hashes_[offset + index], hashes_[offset + index + 1]); - offset += layer_size; - layer_size >>= 1; - index >>= 1; - } - root_ = current; - return root_; -} - -} // namespace bb::stdlib::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.hpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.hpp deleted file mode 100644 index d16e6388c485..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/memory_tree.hpp +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once -#include "hash_path.hpp" - -namespace bb::stdlib::merkle_tree { - -using namespace bb; - -/** - * A MemoryTree is structured as follows: - * hashes_ - * +------------------------------------------------------------------------------+ - * | 0 -> h_{0,0} h_{0,1} h_{0,2} h_{0,3} h_{0,4} h_{0,5} h_{0,6} h_{0,7} | - * i | | - * n | 8 -> h_{1,0} h_{1,1} h_{1,2} h_{1,3} | - * d | | - * e | 12 -> h_{2,0} h_{2,1} | - * x | | - * | 14 -> h_{3,0} | - * +------------------------------------------------------------------------------+ - * - * Here, depth_ = 3 and {h_{0,j}}_{i=0..7} are leaf values. - * Also, root_ = h_{3,0} and total_size_ = (2 * 8 - 2) = 14. - * Lastly, h_{i,j} = hash( h_{i-1,2j}, h_{i-1,2j+1} ) where i > 1. - */ -class MemoryTree { - public: - MemoryTree(size_t depth); - - fr_hash_path get_hash_path(size_t index); - - fr_sibling_path get_sibling_path(size_t index); - - fr update_element(size_t index, fr const& value); - - fr root() const { return root_; } - - public: - size_t depth_; - size_t total_size_; - bb::fr root_; - std::vector hashes_; -}; - -} // namespace bb::stdlib::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.hpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.hpp deleted file mode 100644 index 86e973a18f60..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.hpp +++ /dev/null @@ -1,98 +0,0 @@ -#pragma once -#include "barretenberg/stdlib/primitives/field/field.hpp" -#include "hash_path.hpp" - -namespace bb::stdlib::merkle_tree { - -using namespace bb; - -class MemoryStore; - -template class MerkleTree { - public: - typedef uint256_t index_t; - - MerkleTree(Store& store, size_t depth, uint8_t tree_id = 0); - MerkleTree(MerkleTree const& other) = delete; - MerkleTree(MerkleTree&& other); - ~MerkleTree(); - - fr_hash_path get_hash_path(index_t index); - - fr_sibling_path get_sibling_path(index_t index); - - fr update_element(index_t index, fr const& value); - - fr root() const; - - size_t depth() const { return depth_; } - - index_t size() const; - - protected: - void load_metadata(); - - /** - * Computes the root hash of a tree of `height`, that is empty other than `value` at `index`. - * - * @param height: The tree depth - * @param index: the index of the non-empty leaf - * @param value: the value to be stored in the non-empty leaf - * - * @see Check full documentation: https://hackmd.io/2zyJc6QhRuugyH8D78Tbqg?view - */ - fr update_element(fr const& root, fr const& value, index_t index, size_t height); - - fr get_element(fr const& root, index_t index, size_t height); - - /** - * Computes the root hash of a tree of `height`, that is empty other than `value` at `index`. - * - * @param height: The tree depth - * @param index: the index of the non-empty leaf - * @param value: the value to be stored in the non-empty leaf - */ - fr compute_zero_path_hash(size_t height, index_t index, fr const& value); - - /** - * Given child nodes `a` and `b` and index of `a`, compute their parent node `p` and store [p : (a, b)]. - * - * @param a_index: the index of the child node `a` - * @param a: child node - * @param b: child node - * @param height: the height of the parent node - */ - fr binary_put(index_t a_index, fr const& a, fr const& b, size_t height); - - fr fork_stump( - fr const& value1, index_t index1, fr const& value2, index_t index2, size_t height, size_t stump_height); - - /** - * Stores a parent node and child nodes in the database as [key : (left, right)]. - * - * @param key: The node value to be stored as key - * @param left: the left child node - * @param right: the right child node - */ - void put(fr const& key, fr const& left, fr const& right); - - /** - * Stores a stump [key : (value, index, true)] in the memory. - * The additional byte `true` is to denote this is a stump. - * - * @param key: The node value to be stored as key - * @param value: value of the non-empty leaf in the stump - * @param index: the index of the non-empty leaf in the stump - */ - void put_stump(fr const& key, index_t index, fr const& value); - - void remove(fr const& key); - - protected: - Store& store_; - std::vector zero_hashes_; - size_t depth_; - uint8_t tree_id_; -}; - -} // namespace bb::stdlib::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp deleted file mode 100644 index df0acdccefc8..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include "nullifier_memory_tree.hpp" -#include "../hash.hpp" - -namespace bb::stdlib::merkle_tree { - -NullifierMemoryTree::NullifierMemoryTree(size_t depth) - : MemoryTree(depth) -{ - ASSERT(depth_ >= 1 && depth <= 32); - total_size_ = 1UL << depth_; - hashes_.resize(total_size_ * 2 - 2); - - // Build the entire tree and fill with 0 hashes. - auto current = WrappedNullifierLeaf::zero().hash(); - size_t layer_size = total_size_; - for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { - for (size_t i = 0; i < layer_size; ++i) { - hashes_[offset + i] = current; - } - current = hash_pair_native(current, current); - } - - // Insert the initial leaf at index 0 - auto initial_leaf = WrappedNullifierLeaf(nullifier_leaf{ .value = 0, .nextIndex = 0, .nextValue = 0 }); - leaves_.push_back(initial_leaf); - root_ = update_element(0, initial_leaf.hash()); -} - -fr NullifierMemoryTree::update_element(fr const& value) -{ - // Find the leaf with the value closest and less than `value` - - // If value is 0 we simply append 0 a null NullifierLeaf to the tree - if (value == 0) { - auto zero_leaf = WrappedNullifierLeaf::zero(); - leaves_.push_back(zero_leaf); - return update_element(leaves_.size() - 1, zero_leaf.hash()); - } - - size_t current; - bool is_already_present; - std::tie(current, is_already_present) = find_closest_leaf(leaves_, value); - - nullifier_leaf current_leaf = leaves_[current].unwrap(); - nullifier_leaf new_leaf = { .value = value, - .nextIndex = current_leaf.nextIndex, - .nextValue = current_leaf.nextValue }; - - if (!is_already_present) { - // Update the current leaf to point it to the new leaf - current_leaf.nextIndex = leaves_.size(); - current_leaf.nextValue = value; - - leaves_[current].set(current_leaf); - - // Insert the new leaf with (nextIndex, nextValue) of the current leaf - leaves_.push_back(new_leaf); - } - - // Update the old leaf in the tree - auto old_leaf_hash = current_leaf.hash(); - size_t old_leaf_index = current; - auto root = update_element(old_leaf_index, old_leaf_hash); - - // Insert the new leaf in the tree - auto new_leaf_hash = new_leaf.hash(); - size_t new_leaf_index = is_already_present ? old_leaf_index : leaves_.size() - 1; - root = update_element(new_leaf_index, new_leaf_hash); - - return root; -} - -} // namespace bb::stdlib::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp deleted file mode 100644 index 624daba7ad9c..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp +++ /dev/null @@ -1,92 +0,0 @@ -#pragma once -#include "../hash.hpp" -#include "../memory_tree.hpp" -#include "nullifier_leaf.hpp" - -namespace bb::stdlib::merkle_tree { - -/** - * An NullifierMemoryTree is structured just like a usual merkle tree: - * - * hashes_ - * +------------------------------------------------------------------------------+ - * | 0 -> h_{0,0} h_{0,1} h_{0,2} h_{0,3} h_{0,4} h_{0,5} h_{0,6} h_{0,7} | - * i | | - * n | 8 -> h_{1,0} h_{1,1} h_{1,2} h_{1,3} | - * d | | - * e | 12 -> h_{2,0} h_{2,1} | - * x | | - * | 14 -> h_{3,0} | - * +------------------------------------------------------------------------------+ - * - * Here, depth_ = 3 and {h_{0,j}}_{i=0..7} are leaf values. - * Also, root_ = h_{3,0} and total_size_ = (2 * 8 - 2) = 14. - * Lastly, h_{i,j} = hash( h_{i-1,2j}, h_{i-1,2j+1} ) where i > 1. - * - * 1. Initial state: - * - * # - * - * # # - * - * # # # # - * - * # # # # # # # # - * - * index 0 1 2 3 4 5 6 7 - * - * val 0 0 0 0 0 0 0 0 - * nextIdx 0 0 0 0 0 0 0 0 - * nextVal 0 0 0 0 0 0 0 0 - * - * 2. Add new leaf with value 30 - * - * val 0 30 0 0 0 0 0 0 - * nextIdx 1 0 0 0 0 0 0 0 - * nextVal 30 0 0 0 0 0 0 0 - * - * 3. Add new leaf with value 10 - * - * val 0 30 10 0 0 0 0 0 - * nextIdx 2 0 1 0 0 0 0 0 - * nextVal 10 0 30 0 0 0 0 0 - * - * 4. Add new leaf with value 20 - * - * val 0 30 10 20 0 0 0 0 - * nextIdx 2 0 3 1 0 0 0 0 - * nextVal 10 0 20 30 0 0 0 0 - * - * 5. Add new leaf with value 50 - * - * val 0 30 10 20 50 0 0 0 - * nextIdx 2 4 3 1 0 0 0 0 - * nextVal 10 50 20 30 0 0 0 0 - */ -class NullifierMemoryTree : public MemoryTree { - - public: - NullifierMemoryTree(size_t depth); - - using MemoryTree::get_hash_path; - using MemoryTree::root; - using MemoryTree::update_element; - - fr update_element(fr const& value); - - const std::vector& get_hashes() { return hashes_; } - const WrappedNullifierLeaf get_leaf(size_t index) - { - return (index < leaves_.size()) ? leaves_[index] : WrappedNullifierLeaf::zero(); - } - const std::vector& get_leaves() { return leaves_; } - - protected: - using MemoryTree::depth_; - using MemoryTree::hashes_; - using MemoryTree::root_; - using MemoryTree::total_size_; - std::vector leaves_; -}; - -} // namespace bb::stdlib::merkle_tree \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp b/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp deleted file mode 100644 index 2135f34da3a9..000000000000 --- a/barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/nullifier_tree/nullifier_tree.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once -#include "../hash.hpp" -#include "../merkle_tree.hpp" -#include "nullifier_leaf.hpp" - -namespace bb::stdlib::merkle_tree { - -using namespace bb; - -template class NullifierTree : public MerkleTree { - public: - typedef uint256_t index_t; - - NullifierTree(Store& store, size_t depth, uint8_t tree_id = 0); - NullifierTree(NullifierTree const& other) = delete; - NullifierTree(NullifierTree&& other); - ~NullifierTree(); - - using MerkleTree::get_hash_path; - using MerkleTree::root; - using MerkleTree::size; - using MerkleTree::depth; - - fr update_element(fr const& value); - - private: - using MerkleTree::update_element; - using MerkleTree::get_element; - using MerkleTree::compute_zero_path_hash; - - private: - using MerkleTree::store_; - using MerkleTree::zero_hashes_; - using MerkleTree::depth_; - using MerkleTree::tree_id_; - std::vector leaves; -}; - -} // namespace bb::stdlib::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/stdlib/types/ultra.hpp b/barretenberg/cpp/src/barretenberg/stdlib/types/ultra.hpp index 96bf0d2b2033..e97d63f0e4ba 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/types/ultra.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/types/ultra.hpp @@ -1,11 +1,11 @@ #pragma once +#include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "barretenberg/plonk/composer/ultra_composer.hpp" #include "barretenberg/plonk/proof_system/commitment_scheme/kate_commitment_scheme.hpp" #include "barretenberg/plonk/proof_system/prover/prover.hpp" #include "barretenberg/plonk/proof_system/types/prover_settings.hpp" #include "barretenberg/stdlib/commitment/pedersen/pedersen.hpp" #include "barretenberg/stdlib/encryption/schnorr/schnorr.hpp" -#include "barretenberg/stdlib/merkle_tree/hash_path.hpp" #include "barretenberg/stdlib/primitives/bigfield/bigfield.hpp" #include "barretenberg/stdlib/primitives/biggroup/biggroup.hpp" #include "barretenberg/stdlib/primitives/bit_array/bit_array.hpp" @@ -52,11 +52,6 @@ using cycle_group_ct = stdlib::cycle_group; using bn254 = stdlib::bn254; using secp256k1_ct = stdlib::secp256k1; -namespace merkle_tree { -using namespace stdlib::merkle_tree; -using hash_path = stdlib::merkle_tree::hash_path; -} // namespace merkle_tree - using schnorr_signature_bits = stdlib::schnorr_signature_bits; // Ultra-composer specific types diff --git a/barretenberg/exports.json b/barretenberg/exports.json index f04b2871163d..13425b898cda 100644 --- a/barretenberg/exports.json +++ b/barretenberg/exports.json @@ -35,6 +35,26 @@ ], "isAsync": false }, + { + "functionName": "pedersen_hashes", + "inArgs": [ + { + "name": "inputs_buffer", + "type": "fr::vec_in_buf" + }, + { + "name": "hash_index", + "type": "const uint32_t *" + } + ], + "outArgs": [ + { + "name": "output", + "type": "fr::out_buf" + } + ], + "isAsync": false + }, { "functionName": "pedersen_hash_buffer", "inArgs": [ @@ -55,6 +75,38 @@ ], "isAsync": false }, + { + "functionName": "poseidon_hash", + "inArgs": [ + { + "name": "inputs_buffer", + "type": "fr::vec_in_buf" + } + ], + "outArgs": [ + { + "name": "output", + "type": "fr::out_buf" + } + ], + "isAsync": false + }, + { + "functionName": "poseidon_hashes", + "inArgs": [ + { + "name": "inputs_buffer", + "type": "fr::vec_in_buf" + } + ], + "outArgs": [ + { + "name": "output", + "type": "fr::out_buf" + } + ], + "isAsync": false + }, { "functionName": "blake2s", "inArgs": [ diff --git a/barretenberg/scripts/bindgen.sh b/barretenberg/scripts/bindgen.sh index e3080a848242..e3d5b9b5a70c 100755 --- a/barretenberg/scripts/bindgen.sh +++ b/barretenberg/scripts/bindgen.sh @@ -3,4 +3,4 @@ set -eu #find ./cpp/src -type f -name "c_bind*.hpp" | ./scripts/decls_json.py > exports.json cat ./scripts/c_bind_files.txt | ./scripts/decls_json.py > exports.json -(cd ./ts && yarn ts-node-esm ./src/bindgen/index.ts ../exports.json > ./src/barretenberg_api/index.ts) +(cd ./ts && yarn node --loader ts-node/esm ./src/bindgen/index.ts ../exports.json > ./src/barretenberg_api/index.ts) \ No newline at end of file diff --git a/barretenberg/scripts/c_bind_files.txt b/barretenberg/scripts/c_bind_files.txt index 255fcd4f5ad6..5b6f0ffc5d0b 100644 --- a/barretenberg/scripts/c_bind_files.txt +++ b/barretenberg/scripts/c_bind_files.txt @@ -1,5 +1,6 @@ ./cpp/src/barretenberg/crypto/pedersen_commitment/c_bind.hpp ./cpp/src/barretenberg/crypto/pedersen_hash/c_bind.hpp +./cpp/src/barretenberg/crypto/poseidon2/c_bind.hpp ./cpp/src/barretenberg/crypto/blake2s/c_bind.hpp ./cpp/src/barretenberg/crypto/schnorr/c_bind.hpp ./cpp/src/barretenberg/crypto/aes128/c_bind.hpp diff --git a/barretenberg/ts/src/barretenberg/__snapshots__/poseidon.test.ts.snap b/barretenberg/ts/src/barretenberg/__snapshots__/poseidon.test.ts.snap new file mode 100644 index 000000000000..3af7544b2b80 --- /dev/null +++ b/barretenberg/ts/src/barretenberg/__snapshots__/poseidon.test.ts.snap @@ -0,0 +1,40 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`poseidon sync poseidonHash 1`] = ` +Fr { + "value": Uint8Array [ + 43, + 202, + 235, + 109, + 88, + 187, + 56, + 186, + 247, + 83, + 213, + 140, + 60, + 150, + 97, + 143, + 234, + 130, + 22, + 51, + 69, + 41, + 94, + 180, + 14, + 136, + 52, + 78, + 235, + 12, + 226, + 161, + ], +} +`; diff --git a/barretenberg/ts/src/barretenberg/pedersen.test.ts b/barretenberg/ts/src/barretenberg/pedersen.test.ts index 4b0150ab4db8..42ba13f9c8ba 100644 --- a/barretenberg/ts/src/barretenberg/pedersen.test.ts +++ b/barretenberg/ts/src/barretenberg/pedersen.test.ts @@ -14,6 +14,29 @@ describe('pedersen sync', () => { expect(result).toMatchSnapshot(); }); + it('pedersenHash perf test', () => { + const loops = 1000; + const fields = Array.from({ length: loops * 2 }).map(() => Fr.random()); + const t = new Timer(); + for (let i = 0; i < loops; ++i) { + api.pedersenHash([fields[i * 2], fields[i * 2 + 1]], 0); + } + const us = t.us() / loops; + console.log(`Executed ${loops} hashes at an average ${us}us / hash`); + }); + + it('pedersenHashes perf test', () => { + const loops = 10; + const numHashesPerLoop = 1024; + const fields = Array.from({ length: numHashesPerLoop * 2 }).map(() => Fr.random()); + const t = new Timer(); + for (let i = 0; i < loops; ++i) { + api.pedersenHashes(fields, 0); + } + const us = t.us() / (numHashesPerLoop * loops); + console.log(`Executed ${numHashesPerLoop * loops} hashes at an average ${us}us / hash`); + }); + it('pedersenHashBuffer', () => { const input = Buffer.alloc(123); input.writeUint32BE(321, 0); diff --git a/barretenberg/ts/src/barretenberg/poseidon.test.ts b/barretenberg/ts/src/barretenberg/poseidon.test.ts new file mode 100644 index 000000000000..2647e4778ddc --- /dev/null +++ b/barretenberg/ts/src/barretenberg/poseidon.test.ts @@ -0,0 +1,39 @@ +import { BarretenbergSync } from './index.js'; +import { Timer } from '../benchmark/timer.js'; +import { Fr } from '../types/index.js'; + +describe('poseidon sync', () => { + let api: BarretenbergSync; + + beforeAll(async () => { + api = await BarretenbergSync.new(); + }); + + it('poseidonHash', () => { + const result = api.poseidonHash([new Fr(4n), new Fr(8n)]); + expect(result).toMatchSnapshot(); + }); + + it('poseidonHash perf test', () => { + const loops = 1000; + const fields = Array.from({ length: loops * 2 }).map(() => Fr.random()); + const t = new Timer(); + for (let i = 0; i < loops; ++i) { + api.poseidonHash([fields[i * 2], fields[i * 2 + 1]]); + } + const us = t.us() / loops; + console.log(`Executed ${loops} hashes at an average ${us}us / hash`); + }); + + it('poseidonHashes perf test', () => { + const loops = 10; + const numHashesPerLoop = 1024; + const fields = Array.from({ length: numHashesPerLoop * 2 }).map(() => Fr.random()); + const t = new Timer(); + for (let i = 0; i < loops; ++i) { + api.poseidonHashes(fields); + } + const us = t.us() / (numHashesPerLoop * loops); + console.log(`Executed ${numHashesPerLoop * loops} hashes at an average ${us}us / hash`); + }); +}); diff --git a/barretenberg/ts/src/barretenberg_api/index.ts b/barretenberg/ts/src/barretenberg_api/index.ts index 1a337c33e2ac..f74134d54675 100644 --- a/barretenberg/ts/src/barretenberg_api/index.ts +++ b/barretenberg/ts/src/barretenberg_api/index.ts @@ -39,6 +39,18 @@ export class BarretenbergApi { return out[0]; } + async pedersenHashes(inputsBuffer: Fr[], hashIndex: number): Promise { + const inArgs = [inputsBuffer, hashIndex].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = await this.wasm.callWasmExport( + 'pedersen_hashes', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + async pedersenHashBuffer(inputBuffer: Uint8Array, hashIndex: number): Promise { const inArgs = [inputBuffer, hashIndex].map(serializeBufferable); const outTypes: OutputType[] = [Fr]; @@ -51,6 +63,30 @@ export class BarretenbergApi { return out[0]; } + async poseidonHash(inputsBuffer: Fr[]): Promise { + const inArgs = [inputsBuffer].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = await this.wasm.callWasmExport( + 'poseidon_hash', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + + async poseidonHashes(inputsBuffer: Fr[]): Promise { + const inArgs = [inputsBuffer].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = await this.wasm.callWasmExport( + 'poseidon_hashes', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + async blake2s(data: Uint8Array): Promise { const inArgs = [data].map(serializeBufferable); const outTypes: OutputType[] = [Buffer32]; @@ -551,6 +587,18 @@ export class BarretenbergApiSync { return out[0]; } + pedersenHashes(inputsBuffer: Fr[], hashIndex: number): Fr { + const inArgs = [inputsBuffer, hashIndex].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = this.wasm.callWasmExport( + 'pedersen_hashes', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + pedersenHashBuffer(inputBuffer: Uint8Array, hashIndex: number): Fr { const inArgs = [inputBuffer, hashIndex].map(serializeBufferable); const outTypes: OutputType[] = [Fr]; @@ -563,6 +611,30 @@ export class BarretenbergApiSync { return out[0]; } + poseidonHash(inputsBuffer: Fr[]): Fr { + const inArgs = [inputsBuffer].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = this.wasm.callWasmExport( + 'poseidon_hash', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + + poseidonHashes(inputsBuffer: Fr[]): Fr { + const inArgs = [inputsBuffer].map(serializeBufferable); + const outTypes: OutputType[] = [Fr]; + const result = this.wasm.callWasmExport( + 'poseidon_hashes', + inArgs, + outTypes.map(t => t.SIZE_IN_BYTES), + ); + const out = result.map((r, i) => outTypes[i].fromBuffer(r)); + return out[0]; + } + blake2s(data: Uint8Array): Buffer32 { const inArgs = [data].map(serializeBufferable); const outTypes: OutputType[] = [Buffer32];