From eb96261487c3cb276e81a7bfea1db7b9e485c13b Mon Sep 17 00:00:00 2001 From: dbanks12 Date: Fri, 13 Jan 2023 18:27:40 +0000 Subject: [PATCH 01/10] cmake, dockerfile, constants updates to support dependent projects (native + wasm) --- cpp/dockerfiles/Dockerfile.wasm-linux-clang | 4 +- cpp/dockerfiles/Dockerfile.x86_64-linux-clang | 5 +- cpp/src/aztec/CMakeLists.txt | 54 ++++++++++++++++++- .../aztec/ecc/curves/grumpkin/grumpkin.hpp | 3 ++ cpp/src/aztec/rollup/constants.hpp | 4 +- .../primitives/bigfield/bigfield.fuzzer.hpp | 1 - .../primitives/bit_array/bit_array.fuzzer.hpp | 1 - .../stdlib/primitives/bool/bool.fuzzer.hpp | 1 - .../byte_array/byte_array.fuzzer.hpp | 1 - .../aztec/stdlib/primitives/field/field.cpp | 5 +- .../stdlib/primitives/field/field.fuzzer.hpp | 4 +- .../stdlib/primitives/safe_uint/safe_uint.cpp | 8 +-- .../primitives/safe_uint/safe_uint.fuzzer.hpp | 4 +- .../stdlib/primitives/uint/uint.fuzzer.hpp | 1 - 14 files changed, 75 insertions(+), 21 deletions(-) diff --git a/cpp/dockerfiles/Dockerfile.wasm-linux-clang b/cpp/dockerfiles/Dockerfile.wasm-linux-clang index 7a2d16c126..67029305d3 100644 --- a/cpp/dockerfiles/Dockerfile.wasm-linux-clang +++ b/cpp/dockerfiles/Dockerfile.wasm-linux-clang @@ -5,7 +5,9 @@ WORKDIR /usr/src/barretenberg/cpp/src RUN curl -s -L https://github.com/CraneStation/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz | tar zxfv - WORKDIR /usr/src/barretenberg/cpp COPY . . -RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && make -j$(nproc) barretenberg.wasm +RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && make -j$(nproc) barretenberg env barretenberg.wasm FROM alpine:3.13 COPY --from=builder /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm +COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a /usr/src/barretenberg/cpp/build/bin/lib/libbarretenberg.a +COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libenv.a /usr/src/barretenberg/cpp/build/bin/lib/libenv.a \ No newline at end of file diff --git a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang index 87ac028098..144c480a79 100644 --- a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang +++ b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang @@ -18,7 +18,7 @@ RUN git clone -b release/10.x --depth 1 https://github.com/llvm/llvm-project.git WORKDIR /usr/src/barretenberg/cpp COPY . . # Only build binaries that are needed upstream. -RUN mkdir build && cd build && cmake -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a .. && make -j$(nproc) db_cli rollup_cli tx_factory keygen +RUN mkdir build && cd build && cmake -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a .. && make -j$(nproc) barretenberg db_cli rollup_cli tx_factory keygen FROM alpine:3.13 RUN apk update && apk add llvm10-libs @@ -26,4 +26,5 @@ COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/s COPY --from=builder /usr/src/barretenberg/cpp/build/bin/db_cli /usr/src/barretenberg/cpp/build/bin/db_cli COPY --from=builder /usr/src/barretenberg/cpp/build/bin/rollup_cli /usr/src/barretenberg/cpp/build/bin/rollup_cli COPY --from=builder /usr/src/barretenberg/cpp/build/bin/tx_factory /usr/src/barretenberg/cpp/build/bin/tx_factory -COPY --from=builder /usr/src/barretenberg/cpp/build/bin/keygen /usr/src/barretenberg/cpp/build/bin/keygen \ No newline at end of file +COPY --from=builder /usr/src/barretenberg/cpp/build/bin/keygen /usr/src/barretenberg/cpp/build/bin/keygen +COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a /usr/src/barretenberg/cpp/build/bin/lib/libbarretenberg.a \ No newline at end of file diff --git a/cpp/src/aztec/CMakeLists.txt b/cpp/src/aztec/CMakeLists.txt index c588d7bfb7..eac67df28c 100644 --- a/cpp/src/aztec/CMakeLists.txt +++ b/cpp/src/aztec/CMakeLists.txt @@ -1,4 +1,5 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) add_compile_options(-Werror -Wall -Wextra -Wconversion -Wsign-conversion -Wno-deprecated -Wno-tautological-compare -Wfatal-errors) @@ -62,6 +63,7 @@ if(WASM) $ $ $ + # TODO: remove all except those needed for testing (join_split) $ $ $ @@ -91,5 +93,55 @@ if(WASM) barretenberg.wasm DEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/barretenberg.wasm ) + # For use when compiling dependent cpp projects for WASM + message(STATUS "Compiling all-in-one barretenberg WASM archive") + add_library( + barretenberg + STATIC + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + ) -endif() +else() + # For use when compiling dependent cpp projects + message(STATUS "Compiling all-in-one barretenberg archive") + add_library( + barretenberg + STATIC + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + ) +endif() \ No newline at end of file diff --git a/cpp/src/aztec/ecc/curves/grumpkin/grumpkin.hpp b/cpp/src/aztec/ecc/curves/grumpkin/grumpkin.hpp index d6c2a05d91..918969d6ba 100644 --- a/cpp/src/aztec/ecc/curves/grumpkin/grumpkin.hpp +++ b/cpp/src/aztec/ecc/curves/grumpkin/grumpkin.hpp @@ -5,6 +5,9 @@ #include "../bn254/fr.hpp" namespace grumpkin { + +constexpr size_t MAX_NO_WRAP_INTEGER_BIT_LENGTH = 252; + typedef barretenberg::fr fq; typedef barretenberg::fq fr; diff --git a/cpp/src/aztec/rollup/constants.hpp b/cpp/src/aztec/rollup/constants.hpp index 8e106ab666..1311e47b16 100644 --- a/cpp/src/aztec/rollup/constants.hpp +++ b/cpp/src/aztec/rollup/constants.hpp @@ -2,6 +2,8 @@ #include #include #include +#include + namespace rollup { constexpr size_t DATA_TREE_DEPTH = 32; @@ -9,7 +11,7 @@ constexpr size_t NULL_TREE_DEPTH = 256; constexpr size_t ROOT_TREE_DEPTH = 28; constexpr size_t DEFI_TREE_DEPTH = 30; -constexpr size_t MAX_NO_WRAP_INTEGER_BIT_LENGTH = 252; +constexpr size_t MAX_NO_WRAP_INTEGER_BIT_LENGTH = grumpkin::MAX_NO_WRAP_INTEGER_BIT_LENGTH; constexpr size_t MAX_TXS_BIT_LENGTH = 10; constexpr size_t TX_FEE_BIT_LENGTH = MAX_NO_WRAP_INTEGER_BIT_LENGTH - MAX_TXS_BIT_LENGTH; diff --git a/cpp/src/aztec/stdlib/primitives/bigfield/bigfield.fuzzer.hpp b/cpp/src/aztec/stdlib/primitives/bigfield/bigfield.fuzzer.hpp index cd85b09aa1..54a1658303 100644 --- a/cpp/src/aztec/stdlib/primitives/bigfield/bigfield.fuzzer.hpp +++ b/cpp/src/aztec/stdlib/primitives/bigfield/bigfield.fuzzer.hpp @@ -1,7 +1,6 @@ #include #include #include -#include "../../../rollup/constants.hpp" #include // This is a global variable, so that the execution handling class could alter it and signal to the input tester diff --git a/cpp/src/aztec/stdlib/primitives/bit_array/bit_array.fuzzer.hpp b/cpp/src/aztec/stdlib/primitives/bit_array/bit_array.fuzzer.hpp index 32e3520f57..93d613ab30 100644 --- a/cpp/src/aztec/stdlib/primitives/bit_array/bit_array.fuzzer.hpp +++ b/cpp/src/aztec/stdlib/primitives/bit_array/bit_array.fuzzer.hpp @@ -1,6 +1,5 @@ #include #include -#include "../../../rollup/constants.hpp" #define MAX_ARRAY_SIZE 128 diff --git a/cpp/src/aztec/stdlib/primitives/bool/bool.fuzzer.hpp b/cpp/src/aztec/stdlib/primitives/bool/bool.fuzzer.hpp index 8ae2cc1a7e..e6870e4f67 100644 --- a/cpp/src/aztec/stdlib/primitives/bool/bool.fuzzer.hpp +++ b/cpp/src/aztec/stdlib/primitives/bool/bool.fuzzer.hpp @@ -1,6 +1,5 @@ #include #include -#include "../../../rollup/constants.hpp" // This is a global variable, so that the execution handling class could alter it and signal to the input tester that // the input should fail diff --git a/cpp/src/aztec/stdlib/primitives/byte_array/byte_array.fuzzer.hpp b/cpp/src/aztec/stdlib/primitives/byte_array/byte_array.fuzzer.hpp index 937a3e7ff8..37fab5c0c4 100644 --- a/cpp/src/aztec/stdlib/primitives/byte_array/byte_array.fuzzer.hpp +++ b/cpp/src/aztec/stdlib/primitives/byte_array/byte_array.fuzzer.hpp @@ -1,7 +1,6 @@ #include #include #include -#include "../../../rollup/constants.hpp" #define MAX_ARRAY_SIZE 128 diff --git a/cpp/src/aztec/stdlib/primitives/field/field.cpp b/cpp/src/aztec/stdlib/primitives/field/field.cpp index 1a526e3a49..52472a7976 100644 --- a/cpp/src/aztec/stdlib/primitives/field/field.cpp +++ b/cpp/src/aztec/stdlib/primitives/field/field.cpp @@ -2,9 +2,8 @@ #include #include "../bool/bool.hpp" #include "../composers/composers.hpp" -#include "../../../rollup/constants.hpp" #include "pow.hpp" -#include "../../../rollup/constants.hpp" +#include // #pragma GCC diagnostic ignored "-Wunused-variable" // #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -824,7 +823,7 @@ std::array, 3> field_t::slice(const ui const field_t lo_wit = field_t(witness_t(ctx, lo)); const field_t slice_wit = field_t(witness_t(ctx, slice)); - hi_wit.create_range_constraint(rollup::MAX_NO_WRAP_INTEGER_BIT_LENGTH - uint32_t(msb), + hi_wit.create_range_constraint(grumpkin::MAX_NO_WRAP_INTEGER_BIT_LENGTH - uint32_t(msb), "slice: hi value too large."); lo_wit.create_range_constraint(lsb, "slice: lo value too large."); slice_wit.create_range_constraint(msb_plus_one - lsb, "slice: sliced value too large."); diff --git a/cpp/src/aztec/stdlib/primitives/field/field.fuzzer.hpp b/cpp/src/aztec/stdlib/primitives/field/field.fuzzer.hpp index 13465d2d2c..79b1f3b3da 100644 --- a/cpp/src/aztec/stdlib/primitives/field/field.fuzzer.hpp +++ b/cpp/src/aztec/stdlib/primitives/field/field.fuzzer.hpp @@ -1,8 +1,8 @@ #include #include #include -#include "../../../rollup/constants.hpp" #include +#include #include // This is a global variable, so that the execution handling class could alter it and signal to the input tester @@ -1575,7 +1575,7 @@ template class FieldBase { // Check assert conditions if ((lsb > msb) || (msb > 252) || (static_cast(stack[first_index].f().get_value()) >= - (static_cast(1) << rollup::MAX_NO_WRAP_INTEGER_BIT_LENGTH))) { + (static_cast(1) << grumpkin::MAX_NO_WRAP_INTEGER_BIT_LENGTH))) { return 0; } PRINT_SLICE(first_index, lsb, msb, stack) diff --git a/cpp/src/aztec/stdlib/primitives/safe_uint/safe_uint.cpp b/cpp/src/aztec/stdlib/primitives/safe_uint/safe_uint.cpp index 9962b03c0d..71cf84d874 100644 --- a/cpp/src/aztec/stdlib/primitives/safe_uint/safe_uint.cpp +++ b/cpp/src/aztec/stdlib/primitives/safe_uint/safe_uint.cpp @@ -1,7 +1,7 @@ #include "safe_uint.hpp" #include "../bool/bool.hpp" #include "../composers/composers.hpp" -#include "../../../rollup/constants.hpp" +#include namespace plonk { namespace stdlib { @@ -64,14 +64,14 @@ std::array, 3> safe_uint_t::slice( const uint8_t lsb) const { ASSERT(msb >= lsb); - ASSERT(static_cast(msb) <= rollup::MAX_NO_WRAP_INTEGER_BIT_LENGTH); + ASSERT(static_cast(msb) <= grumpkin::MAX_NO_WRAP_INTEGER_BIT_LENGTH); const safe_uint_t lhs = *this; ComposerContext* ctx = lhs.get_context(); const uint256_t value = uint256_t(get_value()); // This should be caught by the proof itself, but the circuit creator will have now way of knowing where the issue // is - ASSERT(value < (static_cast(1) << rollup::MAX_NO_WRAP_INTEGER_BIT_LENGTH)); + ASSERT(value < (static_cast(1) << grumpkin::MAX_NO_WRAP_INTEGER_BIT_LENGTH)); const auto msb_plus_one = uint32_t(msb) + 1; const auto hi_mask = ((uint256_t(1) << (256 - uint32_t(msb))) - 1); const auto hi = (value >> msb_plus_one) & hi_mask; @@ -88,7 +88,7 @@ std::array, 3> safe_uint_t::slice( slice_wit = safe_uint_t(slice); } else { - hi_wit = safe_uint_t(witness_t(ctx, hi), rollup::MAX_NO_WRAP_INTEGER_BIT_LENGTH - uint32_t(msb), "hi_wit"); + hi_wit = safe_uint_t(witness_t(ctx, hi), grumpkin::MAX_NO_WRAP_INTEGER_BIT_LENGTH - uint32_t(msb), "hi_wit"); lo_wit = safe_uint_t(witness_t(ctx, lo), lsb, "lo_wit"); slice_wit = safe_uint_t(witness_t(ctx, slice), msb_plus_one - lsb, "slice_wit"); } diff --git a/cpp/src/aztec/stdlib/primitives/safe_uint/safe_uint.fuzzer.hpp b/cpp/src/aztec/stdlib/primitives/safe_uint/safe_uint.fuzzer.hpp index 8ebe454d0b..bcf9c49581 100644 --- a/cpp/src/aztec/stdlib/primitives/safe_uint/safe_uint.fuzzer.hpp +++ b/cpp/src/aztec/stdlib/primitives/safe_uint/safe_uint.fuzzer.hpp @@ -1,7 +1,7 @@ #include #include #include -#include "../../../rollup/constants.hpp" +#include // This is a global variable, so that the execution handling class could alter it and signal to the input tester that // the input should fail @@ -1247,7 +1247,7 @@ template class SafeUintFuzzBase { // Check assert conditions if ((lsb > msb) || (msb > 252) || (static_cast(stack[first_index].suint.get_value()) >= - (static_cast(1) << rollup::MAX_NO_WRAP_INTEGER_BIT_LENGTH))) { + (static_cast(1) << grumpkin::MAX_NO_WRAP_INTEGER_BIT_LENGTH))) { return 0; } PRINT_SLICE(first_index, lsb, msb, stack) diff --git a/cpp/src/aztec/stdlib/primitives/uint/uint.fuzzer.hpp b/cpp/src/aztec/stdlib/primitives/uint/uint.fuzzer.hpp index 1fb8f4baf5..2be79622aa 100644 --- a/cpp/src/aztec/stdlib/primitives/uint/uint.fuzzer.hpp +++ b/cpp/src/aztec/stdlib/primitives/uint/uint.fuzzer.hpp @@ -3,7 +3,6 @@ #include #include #include -#include "../../../rollup/constants.hpp" // This is a global variable, so that the execution handling class could alter it and signal to the input tester that // the input should fail bool circuit_should_fail = false; From a38d9b7853b2ea6e1fd5d1d75e26c1dfa16c460d Mon Sep 17 00:00:00 2001 From: dbanks12 Date: Sat, 14 Jan 2023 16:30:04 +0000 Subject: [PATCH 02/10] cmake whitespace --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d750e6e5f4..83f003f258 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -40,7 +40,7 @@ if(FUZZING) if(UNDEFINED_BEHAVIOUR_SANITIZER) set(SANITIZER_OPTIONS ${SANITIZER_OPTIONS} -fsanitize=undefined -fno-sanitize=alignment) endif() - + add_compile_options(-fsanitize=fuzzer-no-link ${SANITIZER_OPTIONS}) set(WASM OFF) From 99394c9846afce811b4b03da6b609f46e50f2ba6 Mon Sep 17 00:00:00 2001 From: dbanks12 Date: Fri, 20 Jan 2023 15:28:04 +0000 Subject: [PATCH 03/10] remove library files from bberg dockerfiles --- cpp/dockerfiles/Dockerfile.wasm-linux-clang | 4 +--- cpp/dockerfiles/Dockerfile.x86_64-linux-clang | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cpp/dockerfiles/Dockerfile.wasm-linux-clang b/cpp/dockerfiles/Dockerfile.wasm-linux-clang index 67029305d3..bc717fcdba 100644 --- a/cpp/dockerfiles/Dockerfile.wasm-linux-clang +++ b/cpp/dockerfiles/Dockerfile.wasm-linux-clang @@ -8,6 +8,4 @@ COPY . . RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && make -j$(nproc) barretenberg env barretenberg.wasm FROM alpine:3.13 -COPY --from=builder /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm -COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a /usr/src/barretenberg/cpp/build/bin/lib/libbarretenberg.a -COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libenv.a /usr/src/barretenberg/cpp/build/bin/lib/libenv.a \ No newline at end of file +COPY --from=builder /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm \ No newline at end of file diff --git a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang index 144c480a79..bc0d6a3b25 100644 --- a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang +++ b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang @@ -26,5 +26,4 @@ COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/s COPY --from=builder /usr/src/barretenberg/cpp/build/bin/db_cli /usr/src/barretenberg/cpp/build/bin/db_cli COPY --from=builder /usr/src/barretenberg/cpp/build/bin/rollup_cli /usr/src/barretenberg/cpp/build/bin/rollup_cli COPY --from=builder /usr/src/barretenberg/cpp/build/bin/tx_factory /usr/src/barretenberg/cpp/build/bin/tx_factory -COPY --from=builder /usr/src/barretenberg/cpp/build/bin/keygen /usr/src/barretenberg/cpp/build/bin/keygen -COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a /usr/src/barretenberg/cpp/build/bin/lib/libbarretenberg.a \ No newline at end of file +COPY --from=builder /usr/src/barretenberg/cpp/build/bin/keygen /usr/src/barretenberg/cpp/build/bin/keygen \ No newline at end of file From 71f7ab7378fdc10eacdfd6c32804c8b66c88b750 Mon Sep 17 00:00:00 2001 From: dbanks12 Date: Mon, 23 Jan 2023 16:37:17 +0000 Subject: [PATCH 04/10] remove library artifacts from docker --- cpp/dockerfiles/Dockerfile.wasm-linux-clang | 2 +- cpp/dockerfiles/Dockerfile.x86_64-linux-clang | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/dockerfiles/Dockerfile.wasm-linux-clang b/cpp/dockerfiles/Dockerfile.wasm-linux-clang index bc717fcdba..b463130766 100644 --- a/cpp/dockerfiles/Dockerfile.wasm-linux-clang +++ b/cpp/dockerfiles/Dockerfile.wasm-linux-clang @@ -5,7 +5,7 @@ WORKDIR /usr/src/barretenberg/cpp/src RUN curl -s -L https://github.com/CraneStation/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz | tar zxfv - WORKDIR /usr/src/barretenberg/cpp COPY . . -RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && make -j$(nproc) barretenberg env barretenberg.wasm +RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && make -j$(nproc) barretenberg.wasm FROM alpine:3.13 COPY --from=builder /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm \ No newline at end of file diff --git a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang index bc0d6a3b25..87ac028098 100644 --- a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang +++ b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang @@ -18,7 +18,7 @@ RUN git clone -b release/10.x --depth 1 https://github.com/llvm/llvm-project.git WORKDIR /usr/src/barretenberg/cpp COPY . . # Only build binaries that are needed upstream. -RUN mkdir build && cd build && cmake -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a .. && make -j$(nproc) barretenberg db_cli rollup_cli tx_factory keygen +RUN mkdir build && cd build && cmake -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a .. && make -j$(nproc) db_cli rollup_cli tx_factory keygen FROM alpine:3.13 RUN apk update && apk add llvm10-libs From 9adf70d6bac1e756df70883994662fdcec920b0e Mon Sep 17 00:00:00 2001 From: dbanks12 Date: Mon, 23 Jan 2023 16:39:05 +0000 Subject: [PATCH 05/10] revert odd no-newline diff of wasm dockerfile --- cpp/dockerfiles/Dockerfile.wasm-linux-clang | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/dockerfiles/Dockerfile.wasm-linux-clang b/cpp/dockerfiles/Dockerfile.wasm-linux-clang index b463130766..7a2d16c126 100644 --- a/cpp/dockerfiles/Dockerfile.wasm-linux-clang +++ b/cpp/dockerfiles/Dockerfile.wasm-linux-clang @@ -8,4 +8,4 @@ COPY . . RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && make -j$(nproc) barretenberg.wasm FROM alpine:3.13 -COPY --from=builder /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm \ No newline at end of file +COPY --from=builder /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm From dcf1b7cebd9f801db4ec34d5a697e7789125523d Mon Sep 17 00:00:00 2001 From: dbanks12 Date: Mon, 23 Jan 2023 21:48:37 +0000 Subject: [PATCH 06/10] use everywhere instead of -j with nprocs --- cpp/dockerfiles/Dockerfile.arm64-linux-gcc | 2 +- cpp/dockerfiles/Dockerfile.wasm-linux-clang | 4 ++-- cpp/dockerfiles/Dockerfile.x86_64-linux-clang | 6 +++--- cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert | 6 +++--- cpp/dockerfiles/Dockerfile.x86_64-linux-gcc | 2 +- cpp/docs/Fuzzing.md | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cpp/dockerfiles/Dockerfile.arm64-linux-gcc b/cpp/dockerfiles/Dockerfile.arm64-linux-gcc index 7a9e7bf7df..ccfbe287ef 100644 --- a/cpp/dockerfiles/Dockerfile.arm64-linux-gcc +++ b/cpp/dockerfiles/Dockerfile.arm64-linux-gcc @@ -1,6 +1,6 @@ FROM aztecprotocol/crosstool-ng-arm64:latest WORKDIR /usr/src/barretenberg/cpp COPY . . -RUN mkdir build && cd build && cmake -DTOOLCHAIN=arm64-linux-gcc .. && make -j$(nproc) +RUN mkdir build && cd build && cmake -DTOOLCHAIN=arm64-linux-gcc .. && cmake --build . --parallel RUN cd build && qemu-aarch64 ./test/barretenberg_tests ENTRYPOINT /bin/bash \ No newline at end of file diff --git a/cpp/dockerfiles/Dockerfile.wasm-linux-clang b/cpp/dockerfiles/Dockerfile.wasm-linux-clang index 7a2d16c126..924083bbab 100644 --- a/cpp/dockerfiles/Dockerfile.wasm-linux-clang +++ b/cpp/dockerfiles/Dockerfile.wasm-linux-clang @@ -5,7 +5,7 @@ WORKDIR /usr/src/barretenberg/cpp/src RUN curl -s -L https://github.com/CraneStation/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz | tar zxfv - WORKDIR /usr/src/barretenberg/cpp COPY . . -RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && make -j$(nproc) barretenberg.wasm +RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && cmake --build . --parallel --target barretenberg.wasm FROM alpine:3.13 -COPY --from=builder /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm +COPY --from=builder /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm \ No newline at end of file diff --git a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang index 87ac028098..19c4018ac0 100644 --- a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang +++ b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang @@ -12,13 +12,13 @@ RUN apk update \ RUN git clone -b release/10.x --depth 1 https://github.com/llvm/llvm-project.git \ && cd llvm-project && mkdir build-openmp && cd build-openmp \ && cmake ../openmp -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLIBOMP_ENABLE_SHARED=OFF \ - && make -j$(nproc) \ - && make install \ + && cmake --build . --parallel \ + && cmake --build . --parallel --target install \ && cd ../.. && rm -rf llvm-project WORKDIR /usr/src/barretenberg/cpp COPY . . # Only build binaries that are needed upstream. -RUN mkdir build && cd build && cmake -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a .. && make -j$(nproc) db_cli rollup_cli tx_factory keygen +RUN mkdir build && cd build && cmake -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a .. && cmake --build . --parallel --target db_cli --target rollup_cli --target tx_factory --target keygen FROM alpine:3.13 RUN apk update && apk add llvm10-libs diff --git a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert index fb9ed4d28b..dec88851a3 100644 --- a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert +++ b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert @@ -12,13 +12,13 @@ RUN apk update \ RUN git clone -b release/10.x --depth 1 https://github.com/llvm/llvm-project.git \ && cd llvm-project && mkdir build-openmp && cd build-openmp \ && cmake ../openmp -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLIBOMP_ENABLE_SHARED=OFF \ - && make -j$(nproc) \ - && make install \ + && cmake --build . --parallel \ + && cmake --build . --parallel --target install \ && cd ../.. && rm -rf llvm-project WORKDIR /usr/src/barretenberg/cpp COPY . . # Build everything to ensure everything builds. All tests will be run from the result of this build. -RUN mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=RelWithAssert -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a -DCI=ON .. && make -j$(nproc) +RUN mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=RelWithAssert -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a -DCI=ON .. && cmake --build . --parallel FROM alpine:3.13 RUN apk update && apk add llvm10-libs curl diff --git a/cpp/dockerfiles/Dockerfile.x86_64-linux-gcc b/cpp/dockerfiles/Dockerfile.x86_64-linux-gcc index 2eeaf4fd9f..b71e6f8b3c 100644 --- a/cpp/dockerfiles/Dockerfile.x86_64-linux-gcc +++ b/cpp/dockerfiles/Dockerfile.x86_64-linux-gcc @@ -9,7 +9,7 @@ RUN apk update \ WORKDIR /usr/src/barretenberg/cpp COPY . . # Build the entire project (not just rollup_cli and db_cli), as we want to check everything builds under gcc. -RUN mkdir build && cd build && cmake -DTOOLCHAIN=x86_64-linux-gcc -DCI=ON .. && make -j$(nproc) +RUN mkdir build && cd build && cmake -DTOOLCHAIN=x86_64-linux-gcc -DCI=ON .. && cmake --build . --parallel FROM alpine:3.13 RUN apk update && apk add libstdc++ libgomp diff --git a/cpp/docs/Fuzzing.md b/cpp/docs/Fuzzing.md index 629e7526ea..d1a4025a92 100644 --- a/cpp/docs/Fuzzing.md +++ b/cpp/docs/Fuzzing.md @@ -82,7 +82,7 @@ Build with coverage instrumentation: mkdir build-coverage/ cd build-coverage/ cmake -DFUZZING=ON -DCMAKE_CXX_FLAGS="-fprofile-instr-generate -fcoverage-mapping" .. -make -j$(nproc) +cmake --build . --parallel ``` Then run the fuzzer on the corpus and generate the HTML coverage reports: From f7b2b42919eb28af0be6d052f8093579e22ab10a Mon Sep 17 00:00:00 2001 From: dbanks12 Date: Mon, 23 Jan 2023 21:50:29 +0000 Subject: [PATCH 07/10] forward all bootstrap args to cmake with --target prefix for each. Use cleaner subshell instead of cd'ing back up after ignition in bootstrap. --- cpp/bootstrap.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cpp/bootstrap.sh b/cpp/bootstrap.sh index a1161e385e..ec8159f669 100755 --- a/cpp/bootstrap.sh +++ b/cpp/bootstrap.sh @@ -26,9 +26,7 @@ else fi # Download ignition transcripts. -cd ./srs_db -./download_ignition.sh 3 -cd .. +(cd ./srs_db && ./download_ignition.sh 3) # Pick native toolchain file. if [ "$OS" == "macos" ]; then @@ -51,7 +49,7 @@ fi # Build native. mkdir -p build && cd build cmake -DCMAKE_BUILD_TYPE=RelWithAssert -DTOOLCHAIN=$TOOLCHAIN .. -make -j$(getconf _NPROCESSORS_ONLN) $@ +cmake --build . --parallel ${@/#/--target } cd .. # Install the webassembly toolchain. @@ -64,4 +62,4 @@ cd .. mkdir -p build-wasm && cd build-wasm cmake -DTOOLCHAIN=wasm-linux-clang .. cmake --build . --parallel --target barretenberg.wasm -cd .. +cd .. \ No newline at end of file From 7b79a3130326bdcfd4b3130ff633d365fa97b02e Mon Sep 17 00:00:00 2001 From: dbanks12 Date: Mon, 23 Jan 2023 21:50:56 +0000 Subject: [PATCH 08/10] allow dependent project to set WASI_SDK_PREFIX and have it be used in wasm-linux-clang.cmake --- cpp/cmake/toolchains/wasm-linux-clang.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/toolchains/wasm-linux-clang.cmake b/cpp/cmake/toolchains/wasm-linux-clang.cmake index f8e7796ff5..d410e8833f 100644 --- a/cpp/cmake/toolchains/wasm-linux-clang.cmake +++ b/cpp/cmake/toolchains/wasm-linux-clang.cmake @@ -9,7 +9,10 @@ set(CMAKE_SYSTEM_VERSION 1) set(CMAKE_SYSTEM_PROCESSOR wasm32) set(triple wasm32-wasi) -set(WASI_SDK_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/src/wasi-sdk-12.0") +if (NOT WASI_SDK_PREFIX) + # can be set by a dependent project + set(WASI_SDK_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/src/wasi-sdk-12.0") +endif() set(CMAKE_C_COMPILER ${WASI_SDK_PREFIX}/bin/clang) set(CMAKE_CXX_COMPILER ${WASI_SDK_PREFIX}/bin/clang++) set(CMAKE_AR ${WASI_SDK_PREFIX}/bin/llvm-ar CACHE STRING "wasi-sdk build") From 357c191bf797068e74fc56c5818599790d7d914b Mon Sep 17 00:00:00 2001 From: dbanks12 Date: Tue, 24 Jan 2023 14:45:02 +0000 Subject: [PATCH 09/10] readme updates for cmake --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ce9a659055..d6f36d56f2 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ The structured reference string contains monomials up to x^{2^20}. This SRS was RUN git clone -b release/10.x --depth 1 https://github.com/llvm/llvm-project.git \ && cd llvm-project && mkdir build-openmp && cd build-openmp \ && cmake ../openmp -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLIBOMP_ENABLE_SHARED=OFF \ - && make -j$(nproc) \ - && make install \ + && cmake --build . --parallel \ + && cmake --build . --parallel --target install \ && cd ../.. && rm -rf llvm-project ``` @@ -45,7 +45,7 @@ If you've installed the C++ Vscode extension you should configure it to format o Each module has its own tests. e.g. To build and run `ecc` tests: ``` -make ecc_tests +cmake --build . --parallel --target ecc_tests ./bin/ecc_tests ``` @@ -102,10 +102,10 @@ To build: ``` mkdir build-wasm && cd build-wasm cmake -DTOOLCHAIN=wasm-linux-clang .. -make barretenberg.wasm +cmake --build . --parallel --target barretenberg.wasm ``` -The resulting wasm binary will be at `./src/aztec/barretenberg.wasm`. +The resulting wasm binary will be at `./build-wasm/bin/barretenberg.wasm`. To run the tests, you'll need to install `wasmtime`. @@ -116,7 +116,7 @@ curl https://wasmtime.dev/install.sh -sSf | bash Tests can be built and run like: ``` -make ecc_tests +cmake --build . --parallel --target ecc_tests wasmtime --dir=.. ./bin/ecc_tests ``` @@ -126,7 +126,7 @@ To build: ``` mkdir build-fuzzing && cd build-fuzzing cmake -DTOOLCHAIN=x86_64-linux-clang -DFUZZING=ON .. -make +cmake --build . ``` Fuzzing build turns off building tests and benchmarks, since they are incompatible with libfuzzer interface. From 7f17b6ed016692a4114d5118d5ccc810d3ed07e9 Mon Sep 17 00:00:00 2001 From: dbanks12 Date: Tue, 24 Jan 2023 15:19:24 +0000 Subject: [PATCH 10/10] update readme to use cmake --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d6f36d56f2..5a1b4d0ec8 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ cd cpp ### Parallelise the build -Make sure your MAKEFLAGS environment variable is set to run jobs equal to number of cores. e.g. `MAKEFLAGS=-j$(nproc)`. +Use the `--parallel` option to `cmake --build ` to parallelize builds. This is roughly equivalent to `make -j$(nproc)` but is more portable. ### Formatting @@ -52,13 +52,13 @@ cmake --build . --parallel --target ecc_tests A shorthand for the above is: ``` -make run_ecc_tests +cmake --build . --parallel --target run_ecc_tests ``` Running the entire suite of tests using `ctest`: ``` -make test +cmake --build . --parallel --target test ``` You can run specific tests, e.g. @@ -72,14 +72,14 @@ You can run specific tests, e.g. Some modules have benchmarks. The build targets are named `_bench`. To build and run, for example `ecc` benchmarks. ``` -make ecc_bench +cmake --build . --parallel --target ecc_bench ./src/aztec/ecc/ecc_bench ``` A shorthand for the above is: ``` -make run_ecc_bench +cmake --build . --parallel --target run_ecc_bench ``` ### CMake Build Options @@ -126,11 +126,11 @@ To build: ``` mkdir build-fuzzing && cd build-fuzzing cmake -DTOOLCHAIN=x86_64-linux-clang -DFUZZING=ON .. -cmake --build . +cmake --build . --parallel ``` Fuzzing build turns off building tests and benchmarks, since they are incompatible with libfuzzer interface. To turn on address sanitizer add `-DADDRESS_SANITIZER=ON`. Note that address sanitizer can be used to explore crashes. Sometimes you might have to specify the address of llvm-symbolizer. You have to do it with `export ASAN_SYMBOLIZER_PATH=`. For undefined behaviour sanitizer `-DUNDEFINED_BEHAVIOUR_SANITIZER=ON`. -Note that the fuzzer can be orders of magnitude slower with ASan (2-3x slower) or UBSan on, so it is best to run a non-sanitized build first, minimize the testcase and then run it for a bit of time with sanitizers. +Note that the fuzzer can be orders of magnitude slower with ASan (2-3x slower) or UBSan on, so it is best to run a non-sanitized build first, minimize the testcase and then run it for a bit of time with sanitizers. \ No newline at end of file