diff --git a/barretenberg/acir_tests/bench_acir_tests.sh b/barretenberg/acir_tests/bench_acir_tests.sh index 8cdc19a7c8f2..9f9bd1dd2e55 100755 --- a/barretenberg/acir_tests/bench_acir_tests.sh +++ b/barretenberg/acir_tests/bench_acir_tests.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash +set -e cd "$(dirname "$0")" +./clone_test_vectors.sh + TEST_NAMES=("$@") THREADS=(1 4 16 32 64) BENCHMARKS=$LOG_FILE @@ -11,7 +14,7 @@ if [[ -z "${LOG_FILE}" ]]; then fi if [ "${#TEST_NAMES[@]}" -eq 0 ]; then - TEST_NAMES=(sha256 ecdsa_secp256k1 ecdsa_secp256r1 schnorr double_verify_proof) + TEST_NAMES=$(find acir_tests/bench_* -maxdepth 0 -type d -printf '%f ') fi for TEST in ${TEST_NAMES[@]}; do @@ -23,13 +26,13 @@ done # Build results into string with \n delimited rows and space delimited values. TABLE_DATA="" for TEST in ${TEST_NAMES[@]}; do - GATE_COUNT=$(jq -r --arg test "$TEST" 'select(.name == "gate_count" and .acir_test == $test) | .value' $BENCHMARKS | uniq) - SUBGROUP_SIZE=$(jq -r --arg test "$TEST" 'select(.name == "subgroup_size" and .acir_test == $test) | .value' $BENCHMARKS | uniq) + GATE_COUNT=$(jq -r --arg test "$TEST" 'select(.eventName == "gate_count" and .acir_test == $test) | .value' $BENCHMARKS | uniq) + SUBGROUP_SIZE=$(jq -r --arg test "$TEST" 'select(.eventName == "subgroup_size" and .acir_test == $test) | .value' $BENCHMARKS | uniq) # Name in col 1, gate count in col 2, subgroup size in col 3. TABLE_DATA+="$TEST $GATE_COUNT $SUBGROUP_SIZE" # Each thread timing in subsequent cols. for HC in "${THREADS[@]}"; do - RESULT=$(cat $BENCHMARKS | jq -r --arg test "$TEST" --argjson hc $HC 'select(.name == "proof_construction_time" and .acir_test == $test and .threads == $hc) | .value') + RESULT=$(cat $BENCHMARKS | jq -r --arg test "$TEST" --argjson hc $HC 'select(.eventName == "proof_construction_time" and .acir_test == $test and .threads == $hc) | .value') TABLE_DATA+=" $RESULT" done TABLE_DATA+=$'\n' diff --git a/noir/Earthfile b/noir/Earthfile index bed6e934b42a..1b01c075ea82 100644 --- a/noir/Earthfile +++ b/noir/Earthfile @@ -235,11 +235,17 @@ barretenberg-acir-benches-bb: WORKDIR /usr/src/barretenberg/acir_tests +bench-acir-bb: + # This target is used for debugging the benchmarking target. + # For precise benchmarks in CI use `bench-publish-acir-bb` + FROM +barretenberg-acir-benches-bb + RUN ./bench_acir_tests.sh + export-bench-acir-bb: ARG EARTHLY_GIT_HASH FROM +barretenberg-acir-benches-bb SAVE IMAGE aztecprotocol/barretenberg-acir-benches:$EARTHLY_GIT_HASH - + bench-publish-acir-bb: ARG PULL_REQUEST ARG BRANCH @@ -256,6 +262,6 @@ bench-publish-acir-bb: END RUN mkdir -p ./log - RUN docker run -v "$(pwd)/log":/log -e LOG_FILE=/log/bench-acir.jsonl --rm aztecprotocol/barretenberg-acir-benches:$AZTEC_DOCKER_TAG ./bench_acir_tests.sh sha256 + RUN docker run -v "$(pwd)/log":/log -e LOG_FILE=/log/bench-acir.jsonl --rm aztecprotocol/barretenberg-acir-benches:$AZTEC_DOCKER_TAG ./bench_acir_tests.sh DO ../+UPLOAD_LOGS --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH diff --git a/noir/noir-repo/test_programs/benchmarks/bench_eddsa_poseidon/Nargo.toml b/noir/noir-repo/test_programs/benchmarks/bench_eddsa_poseidon/Nargo.toml new file mode 100644 index 000000000000..bc2a779f7b2d --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_eddsa_poseidon/Nargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bench_eddsa_poseidon" +version = "0.1.0" +type = "bin" +authors = [""] + +[dependencies] diff --git a/noir/noir-repo/test_programs/benchmarks/bench_eddsa_poseidon/Prover.toml b/noir/noir-repo/test_programs/benchmarks/bench_eddsa_poseidon/Prover.toml new file mode 100644 index 000000000000..8cfb63cd80e0 --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_eddsa_poseidon/Prover.toml @@ -0,0 +1,6 @@ +msg = 789 +pub_key_x = "0x16b051f37589e0dcf4ad3c415c090798c10d3095bedeedabfcc709ad787f3507" +pub_key_y = "0x062800ac9e60839fab9218e5ed9d541f4586e41275f4071816a975895d349a5e" +r8_x = "0x163814666f04c4d2969059a6b63ee26a0f9f0f81bd5957b0796e2e8f4a8a2f06" +r8_y = "0x1255b17d9e4bfb81831625b788f8a1665128079ac4b6c8c3cd1b857666a05a54" +s = "1230930278088778318663840827871215383007447616379808164955640681455510074924" \ No newline at end of file diff --git a/noir/noir-repo/test_programs/benchmarks/bench_eddsa_poseidon/src/main.nr b/noir/noir-repo/test_programs/benchmarks/bench_eddsa_poseidon/src/main.nr new file mode 100644 index 000000000000..31c2f1f2d13d --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_eddsa_poseidon/src/main.nr @@ -0,0 +1,12 @@ +use dep::std::eddsa::{eddsa_poseidon_verify}; + +fn main( + msg: pub Field, + pub_key_x: Field, + pub_key_y: Field, + r8_x: Field, + r8_y: Field, + s: Field +) -> pub bool { + eddsa_poseidon_verify(pub_key_x, pub_key_y, s, r8_x, r8_y, msg) +} \ No newline at end of file diff --git a/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash/Nargo.toml b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash/Nargo.toml new file mode 100644 index 000000000000..9024d9a7301b --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash/Nargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bench_poseidon_hash" +version = "0.1.0" +type = "bin" +authors = [""] + +[dependencies] diff --git a/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash/Prover.toml b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash/Prover.toml new file mode 100644 index 000000000000..66779dea9d7b --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash/Prover.toml @@ -0,0 +1 @@ +input = [1,2] diff --git a/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash/src/main.nr b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash/src/main.nr new file mode 100644 index 000000000000..38adeef6ec75 --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash/src/main.nr @@ -0,0 +1,5 @@ +use dep::std::hash::poseidon; + +fn main(input: [Field; 2]) -> pub Field { + poseidon::bn254::hash_2(input) +} diff --git a/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_100/Nargo.toml b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_100/Nargo.toml new file mode 100644 index 000000000000..b23716b2a203 --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_100/Nargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bench_poseidon_hash_100" +version = "0.1.0" +type = "bin" +authors = [""] + +[dependencies] diff --git a/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_100/Prover.toml b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_100/Prover.toml new file mode 100644 index 000000000000..542b4a08dd6f --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_100/Prover.toml @@ -0,0 +1,102 @@ +input = [ + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], +] \ No newline at end of file diff --git a/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_100/src/main.nr b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_100/src/main.nr new file mode 100644 index 000000000000..fc9a5b7a9705 --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_100/src/main.nr @@ -0,0 +1,12 @@ +use dep::std::hash; + +global SIZE = 100; + +fn main(input: [[Field; 2]; SIZE]) -> pub [Field; SIZE] { + let mut results: [Field; SIZE] = [0; SIZE]; + for i in 0..SIZE { + results[i] = hash::poseidon::bn254::hash_2(input[i]); + } + + results +} \ No newline at end of file diff --git a/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_30/Nargo.toml b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_30/Nargo.toml new file mode 100644 index 000000000000..dbcbc07b1baa --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_30/Nargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bench_poseidon_hash_30" +version = "0.1.0" +type = "bin" +authors = [""] + +[dependencies] diff --git a/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_30/Prover.toml b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_30/Prover.toml new file mode 100644 index 000000000000..7792a9ab8e36 --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_30/Prover.toml @@ -0,0 +1,32 @@ +input = [ + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], +] \ No newline at end of file diff --git a/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_30/src/main.nr b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_30/src/main.nr new file mode 100644 index 000000000000..4d2d94e49463 --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_poseidon_hash_30/src/main.nr @@ -0,0 +1,12 @@ +use dep::std::hash; + +global SIZE = 30; + +fn main(input: [[Field; 2]; SIZE]) -> pub [Field; SIZE] { + let mut results: [Field; SIZE] = [0; SIZE]; + for i in 0..SIZE { + results[i] = hash::poseidon::bn254::hash_2(input[i]); + } + + results +} \ No newline at end of file diff --git a/noir/noir-repo/test_programs/benchmarks/bench_sha256/Nargo.toml b/noir/noir-repo/test_programs/benchmarks/bench_sha256/Nargo.toml new file mode 100644 index 000000000000..488b94ca8586 --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_sha256/Nargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bench_sha256" +version = "0.1.0" +type = "bin" +authors = [""] + +[dependencies] diff --git a/noir/noir-repo/test_programs/benchmarks/bench_sha256/Prover.toml b/noir/noir-repo/test_programs/benchmarks/bench_sha256/Prover.toml new file mode 100644 index 000000000000..66779dea9d7b --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_sha256/Prover.toml @@ -0,0 +1 @@ +input = [1,2] diff --git a/noir/noir-repo/test_programs/benchmarks/bench_sha256/src/main.nr b/noir/noir-repo/test_programs/benchmarks/bench_sha256/src/main.nr new file mode 100644 index 000000000000..fc873fb4afbd --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_sha256/src/main.nr @@ -0,0 +1,5 @@ +use dep::std; + +fn main(input: [u8; 2]) -> pub [u8; 32] { + std::hash::sha256(input) +} \ No newline at end of file diff --git a/noir/noir-repo/test_programs/benchmarks/bench_sha256_100/Nargo.toml b/noir/noir-repo/test_programs/benchmarks/bench_sha256_100/Nargo.toml new file mode 100644 index 000000000000..d0c90d75088e --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_sha256_100/Nargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bench_sha256_100" +version = "0.1.0" +type = "bin" +authors = [""] + +[dependencies] diff --git a/noir/noir-repo/test_programs/benchmarks/bench_sha256_100/Prover.toml b/noir/noir-repo/test_programs/benchmarks/bench_sha256_100/Prover.toml new file mode 100644 index 000000000000..542b4a08dd6f --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_sha256_100/Prover.toml @@ -0,0 +1,102 @@ +input = [ + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], +] \ No newline at end of file diff --git a/noir/noir-repo/test_programs/benchmarks/bench_sha256_100/src/main.nr b/noir/noir-repo/test_programs/benchmarks/bench_sha256_100/src/main.nr new file mode 100644 index 000000000000..d78ca8002d2d --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_sha256_100/src/main.nr @@ -0,0 +1,12 @@ +use dep::std; + +global SIZE = 100; + +fn main(input: [[u8; 2]; SIZE]) -> pub [[u8; 32]; SIZE] { + let mut results: [[u8; 32]; SIZE] = [[0; 32]; SIZE]; + for i in 0..SIZE { + results[i] = std::hash::sha256(input[i]); + } + + results +} \ No newline at end of file diff --git a/noir/noir-repo/test_programs/benchmarks/bench_sha256_30/Nargo.toml b/noir/noir-repo/test_programs/benchmarks/bench_sha256_30/Nargo.toml new file mode 100644 index 000000000000..c1dc76df3942 --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_sha256_30/Nargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bench_sha256_30" +version = "0.1.0" +type = "bin" +authors = [""] + +[dependencies] diff --git a/noir/noir-repo/test_programs/benchmarks/bench_sha256_30/Prover.toml b/noir/noir-repo/test_programs/benchmarks/bench_sha256_30/Prover.toml new file mode 100644 index 000000000000..7792a9ab8e36 --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_sha256_30/Prover.toml @@ -0,0 +1,32 @@ +input = [ + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], + [1,2], +] \ No newline at end of file diff --git a/noir/noir-repo/test_programs/benchmarks/bench_sha256_30/src/main.nr b/noir/noir-repo/test_programs/benchmarks/bench_sha256_30/src/main.nr new file mode 100644 index 000000000000..fa66d6265863 --- /dev/null +++ b/noir/noir-repo/test_programs/benchmarks/bench_sha256_30/src/main.nr @@ -0,0 +1,12 @@ +use dep::std; + +global SIZE = 30; + +fn main(input: [[u8; 2]; SIZE]) -> pub [[u8; 32]; SIZE] { + let mut results: [[u8; 32]; SIZE] = [[0; 32]; SIZE]; + for i in 0..SIZE { + results[i] = std::hash::sha256(input[i]); + } + + results +} \ No newline at end of file diff --git a/noir/noir-repo/test_programs/rebuild.sh b/noir/noir-repo/test_programs/rebuild.sh index 4733bad10c3f..d8577449e186 100755 --- a/noir/noir-repo/test_programs/rebuild.sh +++ b/noir/noir-repo/test_programs/rebuild.sh @@ -47,6 +47,13 @@ for dir in $base_path/*; do dirs_to_process+=("$dir") done +for dir in $current_dir/benchmarks/*; do + if [[ ! -d $dir ]]; then + continue + fi + dirs_to_process+=("$dir") +done + # Process each directory in parallel pids=() if [ -z $NO_PARALLEL ]; then diff --git a/yarn-project/circuit-types/src/stats/metrics.ts b/yarn-project/circuit-types/src/stats/metrics.ts index 8b8859a02d67..58fb1c1aca06 100644 --- a/yarn-project/circuit-types/src/stats/metrics.ts +++ b/yarn-project/circuit-types/src/stats/metrics.ts @@ -26,7 +26,43 @@ export interface Metric { /** Metric definitions to track from benchmarks. */ export const Metrics = [ { - name: 'proof_construction_time_sha256', + name: 'proof_construction_time_sha256_ms', + groupBy: 'threads', + description: 'Time needed to generate a proof of an ACIR program.', + events: ['proof_construction_time'], + }, + { + name: 'proof_construction_time_sha256_30_ms', + groupBy: 'threads', + description: 'Time needed to generate a proof of an ACIR program.', + events: ['proof_construction_time'], + }, + { + name: 'proof_construction_time_sha256_100_ms', + groupBy: 'threads', + description: 'Time needed to generate a proof of an ACIR program.', + events: ['proof_construction_time'], + }, + { + name: 'proof_construction_time_poseidon_hash_ms', + groupBy: 'threads', + description: 'Time needed to generate a proof of an ACIR program.', + events: ['proof_construction_time'], + }, + { + name: 'proof_construction_time_poseidon_hash_30_ms', + groupBy: 'threads', + description: 'Time needed to generate a proof of an ACIR program.', + events: ['proof_construction_time'], + }, + { + name: 'proof_construction_time_poseidon_hash_100_ms', + groupBy: 'threads', + description: 'Time needed to generate a proof of an ACIR program.', + events: ['proof_construction_time'], + }, + { + name: 'proof_construction_time_eddsa_poseidon_ms', groupBy: 'threads', description: 'Time needed to generate a proof of an ACIR program.', events: ['proof_construction_time'], diff --git a/yarn-project/scripts/src/benchmarks/aggregate.ts b/yarn-project/scripts/src/benchmarks/aggregate.ts index f206d02539b0..0d171769cc59 100644 --- a/yarn-project/scripts/src/benchmarks/aggregate.ts +++ b/yarn-project/scripts/src/benchmarks/aggregate.ts @@ -67,8 +67,20 @@ function append( /** Processes an entry with event name 'acir-proof-generated' and updates results */ function processAcirProofGenerated(entry: ProofConstructed, results: BenchmarkCollectedResults) { - if (entry.acir_test === 'sha256') { - append(results, `proof_construction_time_sha256`, entry.threads, entry.value); + if (entry.acir_test === 'bench_sha256') { + append(results, 'proof_construction_time_sha256_ms', entry.threads, entry.value); + } else if (entry.acir_test === 'bench_sha256_30') { + append(results, 'proof_construction_time_sha256_30_ms', entry.threads, entry.value); + } else if (entry.acir_test === 'bench_sha256_100') { + append(results, 'proof_construction_time_sha256_100_ms', entry.threads, entry.value); + } else if (entry.acir_test === 'bench_poseidon_hash') { + append(results, 'proof_construction_time_poseidon_hash_ms', entry.threads, entry.value); + } else if (entry.acir_test === 'bench_poseidon_hash_30') { + append(results, 'proof_construction_time_poseidon_hash_30_ms', entry.threads, entry.value); + } else if (entry.acir_test === 'bench_poseidon_hash_100') { + append(results, 'proof_construction_time_poseidon_hash_100_ms', entry.threads, entry.value); + } else if (entry.acir_test === 'bench_eddsa') { + append(results, 'proof_construction_time_eddsa_poseidon_ms', entry.threads, entry.value); } }