diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 1773ae4..f7b9a83 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -13,9 +13,6 @@ concurrency: group: "${{ github.workflow }}-${{ github.ref }}" cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} -env: - BENCH_RUNS: 10 - jobs: baseline: runs-on: ubuntu-latest @@ -27,73 +24,27 @@ jobs: - uses: actions/checkout@v4 - - name: restore previous baseline results - id: baseline-restore - uses: actions/cache/restore@v4 + - name: cache previous baseline results + uses: actions/cache@v4 with: path: | bench-*.txt - key: ${{ runner.os }}-bench-main - - - name: run current benchmarks - # We have to juggle file names here because the cache action saves and - # restores the same file path and we need to keep it around for - # benchstat comparison before saving new results with the same name. - run: | - make bench | tee bench-main-curr.txt - mv -f bench-main.txt bench-main-prev.txt - cp bench-main-curr.txt bench-main.txt - - CURR_VERSION="${GITHUB_SHA::7}" - CURR_URL="https://github.com/mccutchen/websocket/commit/$CURR_VERSION" - CURR_LINK="[$CURR_VERSION]($CURR_URL)" - - # record commit for which the benchmarks were run - echo -n "$CURR_VERSION" > bench-version.txt - - echo "### benchmarks: $CURR_LINK" >>$GITHUB_STEP_SUMMARY - echo '```' >>$GITHUB_STEP_SUMMARY - cat bench-main-curr.txt >>$GITHUB_STEP_SUMMARY - echo '```' >>$GITHUB_STEP_SUMMARY - - - name: run prev benchmarks if necessary - if: ${{ steps.baseline-restore.outputs.cache-hit != '' }} - run: | - # Determine the base SHA depending on the event type - if [ "${{ github.event_name }}" = "pull_request" ]; then - BASE_SHA=${{ github.event.pull_request.base.sha }} - else - BASE_SHA=$(git rev-parse HEAD~1) - fi + key: ${{ runner.os }}-bench-results-${{ hashFiles('websocket_benchmark_test.go') }} + restore-keys: | + ${{ runner.os }}-bench-results- - git fetch origin main $BASE_SHA - git reset --hard $BASE_SHA - make bench | tee bench-main-prev.txt - git reset --hard $GITHUB_SHA - - # TODO: cache benchstat - - name: compare results with benchstat - id: benchstat + - name: run benchmarks run: | - go run golang.org/x/perf/cmd/benchstat@latest bench-main-prev.txt bench-main-curr.txt | tee -a $GITHUB_OUTPUT bench-stats.txt - - CURR_VERSION="${GITHUB_SHA::7}" - CURR_URL="https://github.com/mccutchen/websocket/commit/$CURR_VERSION" - CURR_LINK="[$CURR_VERSION]($CURR_URL)" - - PREV_VERSION="$(cat bench-version.txt 2>/dev/null)" - PREV_URL="https://github.com/mccutchen/websocket/commit/$PREV_VERSION" - PREV_LINK="[$PREV_VERSION]($PREV_URL)" - - echo "### benchstats: $PREV_LINK (old) vs $CURR_LINK (new)" >>$GITHUB_STEP_SUMMARY - echo '```' >>$GITHUB_STEP_SUMMARY - cat bench-stats.txt >>$GITHUB_STEP_SUMMARY - echo '```' >>$GITHUB_STEP_SUMMARY - - - name: save new baseline results - id: baseline-save - uses: actions/cache/save@v4 - with: - path: | - bench-*.txt - key: ${{ runner.os }}-bench-main + COMMIT="${GITHUB_SHA::7}" + COMMIT_URL="https://github.com/mccutchen/websocket/commit/$COMMIT" + COMMIT_LINK="[$COMMIT]($COMMIT_URL)" + + echo -n "$COMMIT" > bench-commit-main.txt + make bench | tee bench-results-main.txt + + cat <>$GITHUB_STEP_SUMMARY + ### benchmark results @ $COMMIT_LINK + \`\`\` + $(cat bench-results-main.txt) + \`\`\` + EOF diff --git a/Makefile b/Makefile index 0090a23..6359506 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ COVERAGE_PATH ?= coverage.out COVERAGE_ARGS ?= -covermode=atomic -coverprofile=$(COVERAGE_PATH) TEST_ARGS ?= -race -timeout 60s -count=1 -BENCH_COUNT ?= 5 +BENCH_COUNT ?= 10 BENCH_ARGS ?= -bench=. -benchmem -count=$(BENCH_COUNT) # 3rd party tools @@ -56,7 +56,7 @@ testautobahn: bench: go test $(BENCH_ARGS) .PHONY: bench - + lint: test -z "$$($(CMD_GOFUMPT) -d -e .)" || (echo "Error: gofmt failed"; $(CMD_GOFUMPT) -d -e . ; exit 1) go vet ./... diff --git a/websocket_benchmark_test.go b/websocket_benchmark_test.go index e408309..2f5ee22 100644 --- a/websocket_benchmark_test.go +++ b/websocket_benchmark_test.go @@ -28,9 +28,7 @@ func makeFrame(opcode websocket.Opcode, fin bool, payloadLen int) *websocket.Fra func BenchmarkReadFrame(b *testing.B) { frameSizes := []int{ - 256, 1024, - 256 * 1024, 1024 * 1024, // largest cases from the autobahn test suite 8 * 1024 * 1024, @@ -64,13 +62,19 @@ func BenchmarkReadMessage(b *testing.B) { msgSize int frameCount int }{ - {16 * 1024, 4}, - {16 * 1024, 16}, - {1024 * 1024, 4}, - // worst case sizes from autobahn test suite + // 1 frame per message + {1024 * 1024, 1}, {8 * 1024 * 1024, 1}, - {8 * 1024 * 1024, 8}, {16 * 1024 * 1024, 1}, + + // 4 frames per message + {1024 * 1024, 4}, + {8 * 1024 * 1024, 4}, + {16 * 1024 * 1024, 4}, + + // 16 frames per message + {1024 * 1024, 16}, + {8 * 1024 * 1024, 16}, {16 * 1024 * 1024, 16}, }