diff --git a/docs/examples/bootstrap.sh b/docs/examples/bootstrap.sh index eff21032626a..5103fb64bee3 100755 --- a/docs/examples/bootstrap.sh +++ b/docs/examples/bootstrap.sh @@ -6,15 +6,13 @@ REPO_ROOT=$(git rev-parse --show-toplevel) export BB=${BB:-"$REPO_ROOT/barretenberg/cpp/build/bin/bb"} export NARGO=${NARGO:-"$REPO_ROOT/noir/noir-repo/target/release/nargo"} -export TRANSPILER=${TRANSPILER:-"$REPO_ROOT/avm-transpiler/target/release/avm-transpiler"} export STRIP_AZTEC_NR_PREFIX=${STRIP_AZTEC_NR_PREFIX:-"$REPO_ROOT/noir-projects/noir-contracts/scripts/strip_aztec_nr_prefix.sh"} export BB_HASH=${BB_HASH:-$("$REPO_ROOT/barretenberg/cpp/bootstrap.sh" hash)} function compile { echo_header "Compiling example contracts" - # Use noir-contracts bootstrap with DOCS_WORKING_DIR pointing to parent (docs/) - DOCS_WORKING_DIR="$(cd .. && pwd)" \ - $REPO_ROOT/noir-projects/noir-contracts/bootstrap.sh compile "$@" + # Use noir-contracts bootstrap with WORKING_DIR pointing to examples. + WORKING_DIR="docs/examples" $REPO_ROOT/noir-projects/noir-contracts/bootstrap.sh compile "$@" } function compile-solidity { diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 4bd9a5ed653d..34fefe7a17c2 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -1,28 +1,16 @@ #!/usr/bin/env bash -# TODO: THIS SCRIPT SHOULD NOW BE ABLE TO REPLACE TRANSPILATION AND VK GENERATION WITH 'bb aztec_process'. -# # Some notes if you have to work on this script. -# - First of all, I'm sorry (edit: not sorry). It's a beautiful script but it's no fun to debug. I got carried away. # - You can enable BUILD_SYSTEM_DEBUG=1 but the output is quite verbose that it's not much use by default. -# - This flag however, isn't carried into exported functions. You need to do "set -x" in those functions manually. # - You can call ./bootstrap.sh compile to compile and process select contracts. -# - You can disable further parallelism by setting passing 1 as arg to 'parallelize' and with PARALLELISM=1. +# - You can disable parallelism with PARALLELISM=1. # - The exported functions called by parallel must enable their own flags at the start e.g. set -euo pipefail -# - The exported functions are using stdin/stdout, so be very careful about what's printed where. # - The exported functions need to have external variables they require, to have been exported first. -# - You can't export bash arrays or maps to be used by external functions, only strings. # - If you want to echo something, send it to stderr e.g. echo_stderr "My debug" -# - If you call another script, be sure it also doesn't output something you don't want. # - Local assignments with sub-shells don't propagate errors e.g. local capture=$(false). Declare locals separately. -# - Just ask me (charlie) for guidance if you're suffering. -# - I remain convinced we don't need node for these kinds of things, and we can be more performant/expressive with bash. -# - We could perhaps make it less tricky to work with by leveraging more tempfiles and less stdin/stdout. source $(git rev-parse --show-toplevel)/ci3/source_bootstrap # entrypoint for docs -if [ -n "${DOCS_WORKING_DIR:-}" ]; then - cd "$DOCS_WORKING_DIR" -fi +working_dir="${WORKING_DIR:-noir-projects/noir-contracts/contracts}" export RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-16} export HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} @@ -30,105 +18,27 @@ export PLATFORM_TAG=any export BB=${BB:-../../barretenberg/cpp/build/bin/bb} export NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo} -export TRANSPILER=${TRANSPILER:-../../avm-transpiler/target/release/avm-transpiler} export STRIP_AZTEC_NR_PREFIX=${STRIP_AZTEC_NR_PREFIX:-./scripts/strip_aztec_nr_prefix.sh} export BB_HASH=${BB_HASH:-$(../../barretenberg/cpp/bootstrap.sh hash)} export NOIR_HASH=${NOIR_HASH:-$(../../noir/bootstrap.sh hash)} -export tmp_dir=./target/tmp - -# Remove our tmp dir from last run. -# Note: This can use BASH 'trap' for better cleanliness, but the script has been hitting edge-cases so is (temporarily?) simplified. -rm -rf $tmp_dir -mkdir -p $tmp_dir - # Set common flags for parallel. export PARALLEL_FLAGS="-j${PARALLELISM:-16} --halt now,fail=1 --memsuspend $(memsuspend_limit)" -# This computes a vk and adds it to the input function json if it's private, else returns same input. -# stdin has the function json. -# stdout receives the function json with the vk added (if private). -# The function is exported and called by a sub-shell in parallel, so we must "set -eu" etc.. -# If debugging, a set -x at the start can help. -function process_function { - set -euo pipefail - local func name bytecode_b64 hash vk - - contract_hash=$1 - # Read the function json. - func="$(cat)" - name=$(echo "$func" | jq -r '.name') - echo_stderr "Processing function: $name..." - - # Check if the function is neither public nor unconstrained. - # TODO: Why do we need to gen keys for functions that are not marked private? - # We allow the jq call to error (set +e) because it returns an error code if the result is false. - # We then differentiate between a real error, and the result being false. - set +e - make_vk=$(echo "$func" | jq -e '(.custom_attributes | index("public") == null) and (.is_unconstrained == false)') - if [ $? -ne 0 ] && [ "$make_vk" != "false" ]; then - echo_stderr "Failed to check function $name is neither public nor unconstrained." - exit 1 - fi - set -e - - if [ "$make_vk" == "true" ]; then - # It's a private function. - # Build hash, check if in cache. - # If it's in the cache it's extracted to $tmp_dir/$hash - bytecode_b64=$(echo "$func" | jq -r '.bytecode') - hash=$((echo "$BB_HASH"; echo "$bytecode_b64") | sha256sum | tr -d ' -') - - if ! cache_download vk-$contract_hash-$hash.tar.gz >&2; then - # It's not in the cache. Generate the vk file and upload it to the cache. - echo_stderr "Generating vk for function: $name..." - - local outdir=$(mktemp -d -p $tmp_dir) - echo "$bytecode_b64" | base64 -d | gunzip | $BB write_vk --scheme chonk --verifier_type standalone -b - -o $outdir -v - mv $outdir/vk $tmp_dir/$contract_hash/$hash - - cache_upload vk-$contract_hash-$hash.tar.gz $tmp_dir/$contract_hash/$hash - fi - - # Return (echo) json containing the base64 encoded verification key. - vk=$(cat $tmp_dir/$contract_hash/$hash | base64 -w 0) - echo "$func" | jq -c --arg vk "$vk" '. + {verification_key: $vk}' - else - echo_stderr "Function $name is neither public nor unconstrained, skipping." - # Not a private function. Return the original function json. - echo "$func" - fi -} -export -f process_function - # Compute hash for a given contract. # $1 is the contract name, $2 is the folder name (e.g. "contracts" or "examples") function get_contract_hash { local contract_path=$(get_contract_path "$1" "$2") - if [ "$2" = "examples" ]; then - # Called from docs - hash_str \ - $NOIR_HASH \ - $(cache_content_hash \ - ../avm-transpiler/.rebuild_patterns \ - ../barretenberg/cpp/.rebuild_patterns \ - ../barretenberg/ts/.rebuild_patterns \ - "^docs/examples/$contract_path/" \ - "^noir-projects/aztec-nr/" \ - "^noir-projects/noir-protocol-circuits/crates/types/") - else - # Called from noir-contracts - hash_str \ - $NOIR_HASH \ - $(cache_content_hash \ - ../../avm-transpiler/.rebuild_patterns \ - ../../barretenberg/cpp/.rebuild_patterns \ - ../../barretenberg/ts/.rebuild_patterns \ - "^noir-projects/noir-contracts/contracts/$contract_path/" \ - "^noir-projects/aztec-nr/" \ - "^noir-projects/noir-protocol-circuits/crates/types/") - fi + hash_str \ + $NOIR_HASH \ + $(cache_content_hash \ + ../../avm-transpiler/.rebuild_patterns \ + ../../barretenberg/cpp/.rebuild_patterns \ + ../../barretenberg/ts/.rebuild_patterns \ + "^$working_dir/$contract_path/" \ + "^noir-projects/aztec-nr/" \ + "^noir-projects/noir-protocol-circuits/crates/types/") } export -f get_contract_hash @@ -149,7 +59,7 @@ function get_contract_path { else # Just contract name provided (e.g. ecdsa_k_account_contract) contract_path=$(grep -oP "(?<=$folder_name/)[^\"]+/$input" Nargo.toml) - if [ -z "$contract_path" ]; then + if [[ -z $contract_path ]]; then echo "Contract $input not found in Nargo.toml" >&2 exit 1 fi @@ -158,9 +68,9 @@ function get_contract_path { } export -f get_contract_path -# This compiles a noir contract, transpile's public functions, and generates vk's for private functions. +# This compiles a noir contract, transpiles public functions, and generates vk's for private functions. # $1 is the input package name, $2 is the folder name (e.g. "contracts" or "examples") -# On exit it's fully processed json artifact is in the target dir. +# On exit its fully processed json artifact is in the target dir. # The function is exported and called by a sub-shell in parallel, so we must "set -eu" etc.. function compile { set -euo pipefail @@ -175,25 +85,12 @@ function compile { contract_hash=$(get_contract_hash $1 $2) if ! cache_download contract-$contract_hash.tar.gz; then $NARGO compile --package $contract --inliner-aggressiveness 0 --pedantic-solving --deny-warnings - $TRANSPILER $json_path $json_path + # bb aztec_process handles both AVM transpilation and VK generation for private functions. + # It has its own internal VK cache at ~/.bb//vk_cache/ + $BB aztec_process -i $json_path -o $json_path $STRIP_AZTEC_NR_PREFIX $json_path cache_upload contract-$contract_hash.tar.gz $json_path fi - - # We segregate equivalent vk's created by process_function. This was done to narrow down potential edge cases with identical VKs - # reading from cache at the same time. Create this folder up-front. - mkdir -p $tmp_dir/$contract_hash - - # Pipe each contract function, one per line (jq -c), into parallel calls of process_function. - # The returned jsons from process_function are converted back to a json array in the second jq -s call. - # When slurping (-s) in the last jq, we get an array of two elements: - # .[0] is the original json (at $json_path) - # .[1] is the updated functions on stdin (-) - # * merges their fields. - jq -c '.functions[]' $json_path | \ - parallel $PARALLEL_FLAGS --keep-order -N1 --block 8M --pipe process_function $contract_hash | \ - jq -s '{functions: .}' | jq -s '.[0] * {functions: .[1].functions}' $json_path - > $tmp_dir/$filename - mv $tmp_dir/$filename $json_path } export -f compile @@ -201,43 +98,32 @@ export -f compile # Otherwise parse out all relevant contracts from the root Nargo.toml and process them in parallel. function build { echo_stderr "Compiling contracts (bb-hash: $BB_HASH)..." - local folder_name - if [ -n "${DOCS_WORKING_DIR:-}" ]; then - folder_name="examples" - else - folder_name="contracts" - fi + local folder_name=$(basename $working_dir) - if [ "$#" -eq 0 ]; then + if [[ "$#" -eq 0 ]]; then rm -rf target - mkdir -p $tmp_dir local contracts=$(grep -oP "(?<=$folder_name/)[^\"]+" Nargo.toml) else local contracts="$@" fi set +e parallel $PARALLEL_FLAGS --joblog joblog.txt -v --line-buffer --tag compile {} $folder_name ::: ${contracts[@]} - code=$? + local code=$? cat joblog.txt return $code } function test_cmds { local -A cache - local folder_name - if [ -n "${DOCS_WORKING_DIR:-}" ]; then - folder_name="examples" - else - folder_name="contracts" - fi + local folder_name=$(basename $working_dir) # Test bb aztec_process command echo "$BB_HASH noir-projects/scripts/test_aztec_process.sh" - i=0 + local i=0 $NARGO test --list-tests --silence-warnings | sort | while read -r package test; do - port=$((45730 + (i++ % ${NUM_TXES:-1}))) - [ -z "${cache[$package]:-}" ] && cache[$package]=$(get_contract_hash $package $folder_name) + local port=$((45730 + (i++ % ${NUM_TXES:-1}))) + [[ -z "${cache[$package]:-}" ]] && cache[$package]=$(get_contract_hash $package $folder_name) echo "${cache[$package]} noir-projects/scripts/run_test.sh noir-contracts $package $test $port" done }