ci: simplify baseline benchmark job

.github/workflows/bench.yaml

@@ -13,9 +13,6 @@ concurrency:
   group: "${{ github.workflow }}-${{ github.ref }}"
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
-env:
-  BENCH_RUNS: 10
-
 jobs:
   baseline:
     runs-on: ubuntu-latest
@@ -27,73 +24,27 @@ jobs:
 
       - uses: actions/checkout@v4
 
-      - name: restore previous baseline results
-        id: baseline-restore
-        uses: actions/cache/restore@v4
+      - name: cache previous baseline results
+        uses: actions/cache@v4
         with:
           path: |
             bench-*.txt
-          key: ${{ runner.os }}-bench-main
-
-      - name: run current benchmarks
-        # We have to juggle file names here because the cache action saves and
-        # restores the same file path and we need to keep it around for
-        # benchstat comparison before saving new results with the same name.
-        run: |
-          make bench | tee bench-main-curr.txt
-          mv -f            bench-main.txt      bench-main-prev.txt
-          cp               bench-main-curr.txt bench-main.txt
-
-          CURR_VERSION="${GITHUB_SHA::7}"
-          CURR_URL="https://github.com/mccutchen/websocket/commit/$CURR_VERSION"
-          CURR_LINK="[$CURR_VERSION]($CURR_URL)"
-
-          # record commit for which the benchmarks were run
-          echo -n "$CURR_VERSION" > bench-version.txt
-
-          echo "### benchmarks: $CURR_LINK" >>$GITHUB_STEP_SUMMARY
-          echo '```'                        >>$GITHUB_STEP_SUMMARY
-          cat  bench-main-curr.txt          >>$GITHUB_STEP_SUMMARY
-          echo '```'                        >>$GITHUB_STEP_SUMMARY
-
-      - name: run prev benchmarks if necessary
-        if: ${{ steps.baseline-restore.outputs.cache-hit != '' }}
-        run: |
-          # Determine the base SHA depending on the event type
-          if [ "${{ github.event_name }}" = "pull_request" ]; then
-            BASE_SHA=${{ github.event.pull_request.base.sha }}
-          else
-            BASE_SHA=$(git rev-parse HEAD~1)
-          fi
+          key: ${{ runner.os }}-bench-results-${{ hashFiles('websocket_benchmark_test.go') }}
+          restore-keys: |
+            ${{ runner.os }}-bench-results-
 
-          git fetch origin main $BASE_SHA
-          git reset --hard $BASE_SHA
-          make bench | tee bench-main-prev.txt
-          git reset --hard $GITHUB_SHA
-
-      # TODO: cache benchstat
-      - name: compare results with benchstat
-        id: benchstat
+      - name: run benchmarks
         run: |
-          go run golang.org/x/perf/cmd/benchstat@latest bench-main-prev.txt bench-main-curr.txt | tee -a $GITHUB_OUTPUT bench-stats.txt
-
-          CURR_VERSION="${GITHUB_SHA::7}"
-          CURR_URL="https://github.com/mccutchen/websocket/commit/$CURR_VERSION"
-          CURR_LINK="[$CURR_VERSION]($CURR_URL)"
-
-          PREV_VERSION="$(cat bench-version.txt 2>/dev/null)"
-          PREV_URL="https://github.com/mccutchen/websocket/commit/$PREV_VERSION"
-          PREV_LINK="[$PREV_VERSION]($PREV_URL)"
-
-          echo "### benchstats: $PREV_LINK (old) vs $CURR_LINK (new)" >>$GITHUB_STEP_SUMMARY
-          echo '```'                                                  >>$GITHUB_STEP_SUMMARY
-          cat  bench-stats.txt                                        >>$GITHUB_STEP_SUMMARY
-          echo '```'                                                  >>$GITHUB_STEP_SUMMARY
-
-      - name: save new baseline results
-        id: baseline-save
-        uses: actions/cache/save@v4
-        with:
-          path: |
-            bench-*.txt
-          key: ${{ runner.os }}-bench-main
+          COMMIT="${GITHUB_SHA::7}"
+          COMMIT_URL="https://github.com/mccutchen/websocket/commit/$COMMIT"
+          COMMIT_LINK="[$COMMIT]($COMMIT_URL)"
+
+          echo -n "$COMMIT" > bench-commit-main.txt
+          make bench | tee bench-results-main.txt
+
+          cat <<EOF >>$GITHUB_STEP_SUMMARY
+          ### benchmark results @ $COMMIT_LINK
+          \`\`\`
+          $(cat bench-results-main.txt)
+          \`\`\`
+          EOF

Makefile

@@ -2,7 +2,7 @@
 COVERAGE_PATH ?= coverage.out
 COVERAGE_ARGS ?= -covermode=atomic -coverprofile=$(COVERAGE_PATH)
 TEST_ARGS     ?= -race -timeout 60s -count=1
-BENCH_COUNT   ?= 5
+BENCH_COUNT   ?= 10
 BENCH_ARGS    ?= -bench=. -benchmem -count=$(BENCH_COUNT)
 
 # 3rd party tools
@@ -56,7 +56,7 @@ testautobahn:
 bench:
 	go test $(BENCH_ARGS)
 .PHONY: bench
-	
+
 lint:
 	test -z "$$($(CMD_GOFUMPT) -d -e .)" || (echo "Error: gofmt failed"; $(CMD_GOFUMPT) -d -e . ; exit 1)
 	go vet ./...

websocket_benchmark_test.go

@@ -28,9 +28,7 @@ func makeFrame(opcode websocket.Opcode, fin bool, payloadLen int) *websocket.Fra
 
 func BenchmarkReadFrame(b *testing.B) {
 	frameSizes := []int{
-		256,
 		1024,
-		256 * 1024,
 		1024 * 1024,
 		// largest cases from the autobahn test suite
 		8 * 1024 * 1024,
@@ -64,13 +62,19 @@ func BenchmarkReadMessage(b *testing.B) {
 		msgSize    int
 		frameCount int
 	}{
-		{16 * 1024, 4},
-		{16 * 1024, 16},
-		{1024 * 1024, 4},
-		// worst case sizes from autobahn test suite
+		// 1 frame per message
+		{1024 * 1024, 1},
 		{8 * 1024 * 1024, 1},
-		{8 * 1024 * 1024, 8},
 		{16 * 1024 * 1024, 1},
+
+		// 4 frames per message
+		{1024 * 1024, 4},
+		{8 * 1024 * 1024, 4},
+		{16 * 1024 * 1024, 4},
+
+		// 16 frames per message
+		{1024 * 1024, 16},
+		{8 * 1024 * 1024, 16},
 		{16 * 1024 * 1024, 16},
 	}