AztecProtocol · dbanks12 · Dec 12, 2022 · Dec 12, 2022 · Dec 12, 2022 · Dec 12, 2022
diff --git a/README.md b/README.md
@@ -17,8 +17,8 @@ The structured reference string contains monomials up to x^{2^20}. This SRS was
 RUN git clone -b release/10.x --depth 1 https://github.com/llvm/llvm-project.git \
   && cd llvm-project && mkdir build-openmp && cd build-openmp \
   && cmake ../openmp -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLIBOMP_ENABLE_SHARED=OFF \
-  && make -j$(nproc) \
-  && make install \
+  && cmake --build . --parallel \
+  && cmake --build . --parallel --target install \
   && cd ../.. && rm -rf llvm-project
 ```
 
@@ -27,12 +27,13 @@ RUN git clone -b release/10.x --depth 1 https://github.com/llvm/llvm-project.git
 Run the bootstrap script. (The bootstrap script will build both the native and wasm versions of barretenberg)
 
 ```
+cd cpp
 ./bootstrap.sh
 ```
 
 ### Parallelise the build
 
-Make sure your MAKEFLAGS environment variable is set to run jobs equal to number of cores. e.g. `MAKEFLAGS=-j$(nproc)`.
+Use the `--parallel` option to `cmake --build <path>` to parallelize builds. This is roughly equivalent to `make -j$(nproc)` but is more portable.
 
 ### Formatting
 
@@ -44,20 +45,20 @@ If you've installed the C++ Vscode extension you should configure it to format o
 Each module has its own tests. e.g. To build and run `ecc` tests:
 
 ```
-make ecc_tests
+cmake --build . --parallel --target ecc_tests
 ./bin/ecc_tests
 ```
 
 A shorthand for the above is:
 
 ```
-make run_ecc_tests
+cmake --build . --parallel --target run_ecc_tests
 ```
 
 Running the entire suite of tests using `ctest`:
 
 ```
-make test
+cmake --build . --parallel --target test
 ```
 
 You can run specific tests, e.g.
@@ -71,14 +72,14 @@ You can run specific tests, e.g.
 Some modules have benchmarks. The build targets are named `<module_name>_bench`. To build and run, for example `ecc` benchmarks.
 
 ```
-make ecc_bench
+cmake --build . --parallel --target ecc_bench
 ./src/aztec/ecc/ecc_bench
 ```
 
 A shorthand for the above is:
 
 ```
-make run_ecc_bench
+cmake --build . --parallel --target run_ecc_bench
 ```
 
 ### CMake Build Options
@@ -101,10 +102,10 @@ To build:
 ```
 mkdir build-wasm && cd build-wasm
 cmake -DTOOLCHAIN=wasm-linux-clang ..
-make barretenberg.wasm
+cmake --build . --parallel --target barretenberg.wasm
 ```
 
-The resulting wasm binary will be at `./src/aztec/barretenberg.wasm`.
+The resulting wasm binary will be at `./build-wasm/bin/barretenberg.wasm`.
 
 To run the tests, you'll need to install `wasmtime`.
 
@@ -115,7 +116,7 @@ curl https://wasmtime.dev/install.sh -sSf | bash
 Tests can be built and run like:
 
 ```
-make ecc_tests
+cmake --build . --parallel --target ecc_tests
 wasmtime --dir=.. ./bin/ecc_tests
 ```
 
@@ -125,11 +126,11 @@ To build:
 ```
 mkdir build-fuzzing && cd build-fuzzing
 cmake -DTOOLCHAIN=x86_64-linux-clang -DFUZZING=ON ..
-make
+cmake --build . --parallel
 ```
 Fuzzing build turns off building tests and benchmarks, since they are incompatible with libfuzzer interface.
 
 To turn on address sanitizer add `-DADDRESS_SANITIZER=ON`. Note that address sanitizer can be used to explore crashes.
 Sometimes you might have to specify the address of llvm-symbolizer. You have to do it with `export ASAN_SYMBOLIZER_PATH=<PATH_TO_SYMBOLIZER>`.
 For undefined behaviour sanitizer `-DUNDEFINED_BEHAVIOUR_SANITIZER=ON`.
-Note that the fuzzer can be orders of magnitude slower with ASan (2-3x slower) or UBSan on, so it is best to run a non-sanitized build first, minimize the testcase and then run it for a bit of time with sanitizers.  
+Note that the fuzzer can be orders of magnitude slower with ASan (2-3x slower) or UBSan on, so it is best to run a non-sanitized build first, minimize the testcase and then run it for a bit of time with sanitizers.
diff --git a/cpp/.gitignore b/cpp/.gitignore
@@ -1,4 +1,4 @@
-./build*
+build*/
 src/wasi-sdk-*
 src/aztec/plonk/proof_system/proving_key/fixtures
 src/aztec/rollup/proofs/*/fixtures

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -40,11 +40,12 @@ if(FUZZING)
     if(UNDEFINED_BEHAVIOUR_SANITIZER)
         set(SANITIZER_OPTIONS ${SANITIZER_OPTIONS} -fsanitize=undefined  -fno-sanitize=alignment)
     endif()
-    
+
     add_compile_options(-fsanitize=fuzzer-no-link ${SANITIZER_OPTIONS})
 
     set(WASM OFF)
     set(BENCHMARKS OFF)
+    set(MULTITHREADING OFF)
     set(TESTING OFF)
 endif()
 
@@ -68,4 +69,4 @@ include(cmake/gtest.cmake)
 include(cmake/benchmark.cmake)
 include(cmake/module.cmake)
 
-add_subdirectory(src/aztec)
+add_subdirectory(src/aztec)
diff --git a/cpp/bootstrap.sh b/cpp/bootstrap.sh
@@ -1,56 +1,59 @@
 #!/bin/bash
-set -e
+set -eu
 
 # Clean.
 rm -rf ./build
 rm -rf ./build-wasm
+rm -rf ./src/wasi-sdk-*
 
 # Install formatting git hook.
-echo "cd ./barretenberg && ./format.sh staged" > ../.git/hooks/pre-commit
-chmod +x ../.git/hooks/pre-commit
+HOOKS_DIR=$(git rev-parse --git-path hooks)
+# The pre-commit script will live in a barretenberg-specific hooks directory
+# That may be just in the top level of this repository,
+# or may be in a .git/modules/barretenberg subdirectory when this is actually a submodule
+# Either way, running `git rev-parse --show-toplevel` from the hooks directory gives the path to barretenberg
+echo "cd \$(git rev-parse --show-toplevel)/cpp && ./format.sh staged" > $HOOKS_DIR/pre-commit
+chmod +x $HOOKS_DIR/pre-commit
 
 # Determine system.
 if [[ "$OSTYPE" == "darwin"* ]]; then
-    OS=macos
+  OS=macos
 elif [[ "$OSTYPE" == "linux-gnu" ]]; then
-    OS=linux
+  OS=linux
 else
-    echo "Unknown OS: $OSTYPE"
-    exit 1
+  echo "Unknown OS: $OSTYPE"
+  exit 1
 fi
 
 # Download ignition transcripts.
-cd ./srs_db
-./download_ignition.sh 3
-cd ..
+(cd ./srs_db && ./download_ignition.sh 3)
 
 # Pick native toolchain file.
 if [ "$OS" == "macos" ]; then
-    export BREW_PREFIX=$(brew --prefix)
-    # Ensure we have toolchain.
-    if [ ! "$?" -eq 0 ] || [ ! -f "$BREW_PREFIX/opt/llvm/bin/clang++" ]; then
-        echo "Default clang not sufficient. Install homebrew, and then: brew install llvm libomp clang-format"
-        exit 1
-    fi
-    ARCH=$(uname -m)
-    if [ "$ARCH" = "arm64" ]; then
-        TOOLCHAIN=arm-apple-clang
-    else
-        TOOLCHAIN=x86_64-apple-clang
-    fi
+  export BREW_PREFIX=$(brew --prefix)
+  # Ensure we have toolchain.
+  if [ ! "$?" -eq 0 ] || [ ! -f "$BREW_PREFIX/opt/llvm/bin/clang++" ]; then
+    echo "Default clang not sufficient. Install homebrew, and then: brew install llvm libomp clang-format"
+    exit 1
+  fi
+  ARCH=$(uname -m)
+  if [ "$ARCH" = "arm64" ]; then
+    TOOLCHAIN=arm-apple-clang
+  else
+    TOOLCHAIN=x86_64-apple-clang
+  fi
 else
-    TOOLCHAIN=x86_64-linux-clang
+  TOOLCHAIN=x86_64-linux-clang
 fi
 
 # Build native.
 mkdir -p build && cd build
 cmake -DCMAKE_BUILD_TYPE=RelWithAssert -DTOOLCHAIN=$TOOLCHAIN ..
-make -j$(getconf _NPROCESSORS_ONLN) $@
+cmake --build . --parallel ${@/#/--target }
 cd ..
 
 # Install the webassembly toolchain.
 WASI_VERSION=12
-rm -rf ./src/wasi-sdk-*
 cd ./src
 curl -s -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$WASI_VERSION/wasi-sdk-$WASI_VERSION.0-$OS.tar.gz | tar zxfv -
 cd ..
@@ -59,4 +62,4 @@ cd ..
 mkdir -p build-wasm && cd build-wasm
 cmake -DTOOLCHAIN=wasm-linux-clang ..
 cmake --build . --parallel --target barretenberg.wasm
-cd ..
+cd ..
diff --git a/cpp/cmake/arch.cmake b/cpp/cmake/arch.cmake
@@ -6,5 +6,5 @@ if(WASM)
 endif()
 
 if(NOT WASM AND NOT APPLE)
-    add_compile_options(-march=skylake-avx512)
+    add_compile_options(-march=skylake)
 endif()
diff --git a/cpp/cmake/toolchains/wasm-linux-clang.cmake b/cpp/cmake/toolchains/wasm-linux-clang.cmake
@@ -9,7 +9,10 @@ set(CMAKE_SYSTEM_VERSION 1)
 set(CMAKE_SYSTEM_PROCESSOR wasm32)
 set(triple wasm32-wasi)
 
-set(WASI_SDK_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/src/wasi-sdk-12.0")
+if (NOT WASI_SDK_PREFIX)
+  # can be set by a dependent project
+  set(WASI_SDK_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/src/wasi-sdk-12.0")
+endif()
 set(CMAKE_C_COMPILER ${WASI_SDK_PREFIX}/bin/clang)
 set(CMAKE_CXX_COMPILER ${WASI_SDK_PREFIX}/bin/clang++)
 set(CMAKE_AR ${WASI_SDK_PREFIX}/bin/llvm-ar CACHE STRING "wasi-sdk build")

diff --git a/cpp/dockerfiles/Dockerfile.arm64-linux-gcc b/cpp/dockerfiles/Dockerfile.arm64-linux-gcc
@@ -1,6 +1,6 @@
 FROM aztecprotocol/crosstool-ng-arm64:latest
-WORKDIR /usr/src/barretenberg
+WORKDIR /usr/src/barretenberg/cpp
 COPY . .
-RUN mkdir build && cd build && cmake -DTOOLCHAIN=arm64-linux-gcc .. && make -j$(nproc)
+RUN mkdir build && cd build && cmake -DTOOLCHAIN=arm64-linux-gcc .. && cmake --build . --parallel
 RUN cd build && qemu-aarch64 ./test/barretenberg_tests
 ENTRYPOINT /bin/bash
diff --git a/cpp/dockerfiles/Dockerfile.wasm-linux-clang b/cpp/dockerfiles/Dockerfile.wasm-linux-clang
@@ -1,11 +1,11 @@
 FROM ubuntu:focal AS builder
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential wget git libssl-dev cmake curl binaryen
 RUN curl https://wasmtime.dev/install.sh -sSf | bash /dev/stdin --version v0.25.0
-WORKDIR /usr/src/barretenberg/src
+WORKDIR /usr/src/barretenberg/cpp/src
 RUN curl -s -L https://github.com/CraneStation/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz | tar zxfv -
-WORKDIR /usr/src/barretenberg
+WORKDIR /usr/src/barretenberg/cpp
 COPY . .
-RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && make -j$(nproc) barretenberg.wasm
+RUN mkdir build && cd build && cmake -DTOOLCHAIN=wasm-linux-clang .. && cmake --build . --parallel --target barretenberg.wasm
 
 FROM alpine:3.13
-COPY --from=builder /usr/src/barretenberg/build/bin/barretenberg.wasm /usr/src/barretenberg/build/bin/barretenberg.wasm
+COPY --from=builder /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm /usr/src/barretenberg/cpp/build/bin/barretenberg.wasm
diff --git a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang
@@ -12,18 +12,18 @@ RUN apk update \
 RUN git clone -b release/10.x --depth 1 https://github.com/llvm/llvm-project.git \
   && cd llvm-project && mkdir build-openmp && cd build-openmp \
   && cmake ../openmp -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLIBOMP_ENABLE_SHARED=OFF \
-  && make -j$(nproc) \
-  && make install \
+  && cmake --build . --parallel \
+  && cmake --build . --parallel --target install \
   && cd ../.. && rm -rf llvm-project
-WORKDIR /usr/src/barretenberg
+WORKDIR /usr/src/barretenberg/cpp
 COPY . .
 # Only build binaries that are needed upstream.
-RUN mkdir build && cd build && cmake -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a .. && make -j$(nproc) db_cli rollup_cli tx_factory keygen
+RUN mkdir build && cd build && cmake -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a .. && cmake --build . --parallel --target db_cli --target rollup_cli --target tx_factory --target keygen
 
 FROM alpine:3.13
 RUN apk update && apk add llvm10-libs
-COPY --from=builder /usr/src/barretenberg/srs_db /usr/src/barretenberg/srs_db
-COPY --from=builder /usr/src/barretenberg/build/bin/db_cli /usr/src/barretenberg/build/bin/db_cli
-COPY --from=builder /usr/src/barretenberg/build/bin/rollup_cli /usr/src/barretenberg/build/bin/rollup_cli
-COPY --from=builder /usr/src/barretenberg/build/bin/tx_factory /usr/src/barretenberg/build/bin/tx_factory
-COPY --from=builder /usr/src/barretenberg/build/bin/keygen /usr/src/barretenberg/build/bin/keygen
+COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db
+COPY --from=builder /usr/src/barretenberg/cpp/build/bin/db_cli /usr/src/barretenberg/cpp/build/bin/db_cli
+COPY --from=builder /usr/src/barretenberg/cpp/build/bin/rollup_cli /usr/src/barretenberg/cpp/build/bin/rollup_cli
+COPY --from=builder /usr/src/barretenberg/cpp/build/bin/tx_factory /usr/src/barretenberg/cpp/build/bin/tx_factory
+COPY --from=builder /usr/src/barretenberg/cpp/build/bin/keygen /usr/src/barretenberg/cpp/build/bin/keygen
diff --git a/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert b/cpp/dockerfiles/Dockerfile.x86_64-linux-clang-assert
@@ -12,15 +12,15 @@ RUN apk update \
 RUN git clone -b release/10.x --depth 1 https://github.com/llvm/llvm-project.git \
   && cd llvm-project && mkdir build-openmp && cd build-openmp \
   && cmake ../openmp -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLIBOMP_ENABLE_SHARED=OFF \
-  && make -j$(nproc) \
-  && make install \
+  && cmake --build . --parallel \
+  && cmake --build . --parallel --target install \
   && cd ../.. && rm -rf llvm-project
-WORKDIR /usr/src/barretenberg
+WORKDIR /usr/src/barretenberg/cpp
 COPY . .
 # Build everything to ensure everything builds. All tests will be run from the result of this build.
-RUN mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=RelWithAssert -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a -DCI=ON .. && make -j$(nproc)
+RUN mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=RelWithAssert -DOpenMP_omp_LIBRARY=/usr/local/lib/libomp.a -DCI=ON .. && cmake --build . --parallel
 
 FROM alpine:3.13
 RUN apk update && apk add llvm10-libs curl
-COPY --from=builder /usr/src/barretenberg/srs_db /usr/src/barretenberg/srs_db
-COPY --from=builder /usr/src/barretenberg/build/bin/*_tests /usr/src/barretenberg/build/bin/
+COPY --from=builder /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db
+COPY --from=builder /usr/src/barretenberg/cpp/build/bin/*_tests /usr/src/barretenberg/cpp/build/bin/
diff --git a/cpp/dockerfiles/Dockerfile.x86_64-linux-gcc b/cpp/dockerfiles/Dockerfile.x86_64-linux-gcc
@@ -6,12 +6,12 @@ RUN apk update \
         cmake \
         git \
         curl
-WORKDIR /usr/src/barretenberg
+WORKDIR /usr/src/barretenberg/cpp
 COPY . .
 # Build the entire project (not just rollup_cli and db_cli), as we want to check everything builds under gcc.
-RUN mkdir build && cd build && cmake -DTOOLCHAIN=x86_64-linux-gcc -DCI=ON .. && make -j$(nproc)
+RUN mkdir build && cd build && cmake -DTOOLCHAIN=x86_64-linux-gcc -DCI=ON .. && cmake --build . --parallel
 
 FROM alpine:3.13
 RUN apk update && apk add libstdc++ libgomp
-COPY --from=builder /usr/src/barretenberg/build/bin/db_cli /usr/src/barretenberg/build/bin/db_cli
-COPY --from=builder /usr/src/barretenberg/build/bin/rollup_cli /usr/src/barretenberg/build/bin/rollup_cli
+COPY --from=builder /usr/src/barretenberg/cpp/build/bin/db_cli /usr/src/barretenberg/cpp/build/bin/db_cli
+COPY --from=builder /usr/src/barretenberg/cpp/build/bin/rollup_cli /usr/src/barretenberg/cpp/build/bin/rollup_cli
diff --git a/cpp/docs/Fuzzing.md b/cpp/docs/Fuzzing.md
@@ -82,7 +82,7 @@ Build with coverage instrumentation:
 mkdir build-coverage/
 cd build-coverage/
 cmake -DFUZZING=ON -DCMAKE_CXX_FLAGS="-fprofile-instr-generate -fcoverage-mapping" ..
-make -j$(nproc)
+cmake --build . --parallel
 ```
 
 Then run the fuzzer on the corpus and generate the HTML coverage reports:

diff --git a/cpp/scripts/run_tests b/cpp/scripts/run_tests
@@ -16,7 +16,7 @@ fi
 
 docker run --rm -t $IMAGE_URI /bin/sh -c "\
   set -e; \
-  cd /usr/src/barretenberg/srs_db; \
+  cd /usr/src/barretenberg/cpp/srs_db; \
   ./download_ignition.sh 1; \
-  cd /usr/src/barretenberg/build; \
+  cd /usr/src/barretenberg/cpp/build; \
   for BIN in $TESTS; do ./bin/\$BIN $@; done"