diff --git a/.github/workflows/ci3.yml b/.github/workflows/ci3.yml index 9f4680862450..6447cf04d21f 100644 --- a/.github/workflows/ci3.yml +++ b/.github/workflows/ci3.yml @@ -192,6 +192,22 @@ jobs: alert-comment-cc-users: "@dbanks12" max-items-in-chart: 50 + - name: Store l1 gas benchmark result + if: matrix.settings.arch == 'amd64' && github.event_name == 'push' && github.ref_name == 'master' + uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 + with: + name: "L1 Gas Benchmark" + benchmark-data-dir-path: "dev/l1-gas-bench" + tool: "customSmallerIsBetter" + output-file-path: ./bench-out/l1-gas-bench.json + github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + auto-push: true + alert-threshold: "110%" + comment-on-alert: true + fail-on-alert: false + alert-comment-cc-users: "@LHerskind" + max-items-in-chart: 50 + # - name: Store p2p benchmark result # if: matrix.settings.arch == 'amd64' && github.event_name == 'push' && github.ref_name == 'master' # continue-on-error: true diff --git a/bootstrap.sh b/bootstrap.sh index 4e4b23829450..ca28acebb28b 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -246,6 +246,7 @@ function bench { denoise "barretenberg/bootstrap.sh bench" denoise "noir-projects/noir-protocol-circuits/bootstrap.sh bench" denoise "yarn-project/simulator/bootstrap.sh bench" + denoise "l1-contracts/bootstrap.sh bench" denoise "yarn-project/end-to-end/bootstrap.sh bench" # denoise "yarn-project/p2p/bootstrap.sh bench" } diff --git a/ci.sh b/ci.sh index 6f19eb70dda2..4d3b577b36e4 100755 --- a/ci.sh +++ b/ci.sh @@ -254,6 +254,7 @@ case "$cmd" in npc_hash=$(git rev-list -n 1 ${AZTEC_CACHE_COMMIT:-HEAD}) # Simulator benchmarks are published on each commit sim_hash=$(git rev-list -n 1 ${AZTEC_CACHE_COMMIT:-HEAD}) + l1_hash=$(l1-contracts/bootstrap.sh hash) yp_hash=$(yarn-project/bootstrap.sh hash) if [ "$bb_hash" == disabled-cache ] || [ "$yp_hash" == disabled-cache ]; then @@ -277,6 +278,8 @@ case "$cmd" in cache_download noir-protocol-circuits-bench-results-$npc_hash.tar.gz # aztec simulator benchmarks. cache_download simulator-bench-results-$sim_hash.tar.gz + # L1 gas benchmark + cache_download l1-gas-bench-results-$l1_hash.tar.gz # yarn-project benchmarks. if [ "$yp_hash" == "$prev_yp_hash" ]; then diff --git a/l1-contracts/.gitignore b/l1-contracts/.gitignore index b7061a4e3112..72edb62f92cc 100644 --- a/l1-contracts/.gitignore +++ b/l1-contracts/.gitignore @@ -31,3 +31,5 @@ gas_report.diff gas_benchmark.new.* gas_benchmark.diff + +/bench-out diff --git a/l1-contracts/README.md b/l1-contracts/README.md index a2aa32b0933a..cb9437d11256 100644 --- a/l1-contracts/README.md +++ b/l1-contracts/README.md @@ -36,6 +36,8 @@ We use `forge fmt` to format. But follow a few general guidelines beyond the sta You can run `./bootstrap.sh gas_report` to generate a detailed gas report for the current state and update the gas_report.md file. +If you want something more manageble you should be using the `./boostrap.sh gas_benchmark` which will give you some "happy path" gas numbers for set with and without validators in a format that might be slightly simpler to figure out. The values outputted from this can also be seen over time at https://aztecprotocol.github.io/aztec-packages/dev/l1-gas-bench/. + When running CI or tests with `./bootstrap.sh test`, the script will automatically check if gas usage has changed by running `./bootstrap.sh gas_report check`. If gas usage has changed, the test will fail and show a diff of the changes. If the changes in gas usage are expected and desired: diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index 68b5016a85d2..979ad1f58f6f 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -131,6 +131,82 @@ function gas_report { mv gas_report.new.json gas_report.json } +function bench { + rm -rf bench-out && mkdir -p bench-out + if cache_download l1-gas-bench-results-$hash.tar.gz; then + return + fi + + # Run the gas benchmark to generate the markdown file + gas_benchmark + + # Extract gas values from gas_benchmark.md and create JSON output + awk ' + function trim(s) { + sub(/^[ \t]+/, "", s); + sub(/[ \t]+$/, "", s); + return s; + } + BEGIN { + print "["; + first = 1; + } + /^[a-zA-Z]/ { + if ($1 != "Function" && $1 != "-------------------------") { + # Split the line into columns and clean them + n = split($0, cols, "|"); + for (i = 1; i <= n; i++) { + cols[i] = trim(cols[i]); + } + + # Only process Max rows + if (cols[2] == "Max") { + # Get the function name + func_name = cols[1]; + + # Define our cases with their column numbers + cases["no_validators"] = 3; + cases["100_validators"] = 4; + cases["overhead"] = 5; + + for (case_name in cases) { + col = cases[case_name]; + if (match(cols[col], /([0-9]+)[ ]*\(([0-9.]+)\)/)) { + # Extract the raw gas value (first number) + match(cols[col], /[0-9]+/); + raw_gas = substr(cols[col], RSTART, RLENGTH); + + # Extract the per tx value + match(cols[col], /\(([0-9.]+)\)/); + per_tx = substr(cols[col], RSTART+1, RLENGTH-2); + + if (!first) print ","; + first = 0; + + # Output raw gas value + print " {"; + print " \"name\": \"" func_name " (" case_name ")\","; + print " \"value\": " raw_gas ","; + print " \"unit\": \"gas\""; + print " },"; + + # Output per tx value + print " {"; + print " \"name\": \"" func_name " (" case_name ") per l2 tx\","; + print " \"value\": " per_tx ","; + print " \"unit\": \"gas\""; + print " }"; + } + } + } + } + } + END { + print "]"; + }' gas_benchmark.md > ./bench-out/l1-gas-bench.json + + cache_upload l1-gas-bench-results-$hash.tar.gz ./bench-out/l1-gas-bench.json +} function gas_benchmark { check=${1:-"no"} @@ -370,6 +446,9 @@ case "$cmd" in "hash") echo $hash ;; + "bench") + bench + ;; *) echo "Unknown command: $cmd" exit 1