Skip to content

Commit 96b55f4

Browse files
mccutchenmccutchen
authored
Merge 0b59dcb into 6b3cbf1 
2 parents 6b3cbf1 + 0b59dcb commit 96b55f4

File tree

3 files changed

+102
-47
lines changed

3 files changed

+102
-47
lines changed

‎.github/workflows/bench.yaml‎

Lines changed: 87 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ env:
1919
jobs:
2020
baseline:
2121
runs-on: ubuntu-latest
22-
if: ${{ github.ref == 'refs/heads/main' }}
22+
# if: ${{ github.ref == 'refs/heads/main' }}
2323
steps:
2424
- uses: actions/setup-go@v5
2525
with:
@@ -33,67 +33,116 @@ jobs:
3333
with:
3434
path: |
3535
bench-*.txt
36-
key: ${{ runner.os }}-bench-main
36+
key: ${{ runner.os }}-bench-results-${{ hashFiles('websocket_benchmark_test.go') }}
3737

38-
- name: run current benchmarks
39-
# We have to juggle file names here because the cache action saves and
40-
# restores the same file path and we need to keep it around for
41-
# benchstat comparison before saving new results with the same name.
38+
- name: run benchmarks
4239
run: |
43-
make bench | tee bench-main-curr.txt
44-
mv -f bench-main.txt bench-main-prev.txt
45-
cp bench-main-curr.txt bench-main.txt
46-
4740
CURR_VERSION="${GITHUB_SHA::7}"
4841
CURR_URL="https://github.com/mccutchen/websocket/commit/$CURR_VERSION"
4942
CURR_LINK="[$CURR_VERSION]($CURR_URL)"
5043
5144
# record commit for which the benchmarks were run
52-
echo -n "$CURR_VERSION" > bench-version.txt
45+
echo -n "$CURR_VERSION" > bench-version-curr.txt
46+
47+
make bench | tee bench-main-curr.txt
5348
49+
# report results
5450
echo "### benchmarks: $CURR_LINK" >>$GITHUB_STEP_SUMMARY
5551
echo '```' >>$GITHUB_STEP_SUMMARY
5652
cat bench-main-curr.txt >>$GITHUB_STEP_SUMMARY
5753
echo '```' >>$GITHUB_STEP_SUMMARY
5854
59-
- name: run prev benchmarks if necessary
60-
if: ${{ steps.baseline-restore.outputs.cache-hit != '' }}
55+
echo "XXX summary:"
56+
cat $GITHUB_STEP_SUMMARY
57+
58+
- name: save new baseline results
59+
uses: actions/cache/save@v4
60+
with:
61+
path: |
62+
bench-*.txt
63+
key: ${{ runner.os }}-bench-results-${{ hashFiles('websocket_benchmark_test.go') }}
64+
65+
pr:
66+
runs-on: ubuntu-latest
67+
if: ${{ github.ref != 'refs/heads/main' }}
68+
permissions:
69+
pull-requests: write
70+
steps:
71+
- uses: actions/setup-go@v5
72+
with:
73+
go-version: "stable"
74+
75+
- uses: actions/checkout@v4
76+
77+
- name: restore previous baseline results
78+
id: baseline-restore
79+
uses: actions/cache/restore@v4
80+
with:
81+
path: |
82+
bench-*.txt
83+
key: ${{ runner.os }}-bench-results-${{ hashFiles('websocket_benchmark_test.go') }}
84+
85+
- name: benchmark current commit
86+
# We have to juggle file names here because the cache action saves and
87+
# restores the same file path and we need to keep it around for
88+
# benchstat comparison before saving new results with the same name.
6189
run: |
62-
# Determine the base SHA depending on the event type
63-
if [ "${{ github.event_name }}" = "pull_request" ]; then
64-
BASE_SHA=${{ github.event.pull_request.base.sha }}
65-
else
66-
BASE_SHA=$(git rev-parse HEAD~1)
90+
if [ -f bench-main-curr.txt ]; then
91+
mv bench-main-curr.txt bench-main-prev.txt
92+
fi
93+
if [ -f bench-version-curr.txt ]; then
94+
mv bench-version-curr.txt bench-version-prev.txt
6795
fi
6896
97+
make bench | tee bench-main-curr.txt
98+
99+
CURR_VERSION="${GITHUB_SHA::7}"
100+
CURR_URL="https://github.com/mccutchen/websocket/commit/$CURR_VERSION"
101+
CURR_LINK="[$CURR_VERSION]($CURR_URL)"
102+
103+
# record commit for which the benchmarks were run
104+
echo -n "$CURR_VERSION" > bench-version-curr.txt
105+
106+
# report results
107+
echo "### benchmarks: $CURR_LINK" >>pr_comment
108+
echo '```' >>pr_comment
109+
cat bench-main-curr.txt >>pr_comment
110+
echo '```' >>pr_comment
111+
cat pr_comment >> $GITHUB_STEP_SUMMARY
112+
113+
- name: benchmark prev commit if necessary
114+
if: ${{ steps.baseline-restore.outputs.cache-hit == '' }}
115+
run: |
116+
BASE_SHA=${{ github.event.pull_request.base.sha }}
117+
echo $BASE_SHA > bench-version-prev.txt
118+
69119
git fetch origin main $BASE_SHA
70120
git reset --hard $BASE_SHA
71121
make bench | tee bench-main-prev.txt
72122
git reset --hard $GITHUB_SHA
73123
74124
# TODO: cache benchstat
75125
- name: compare results with benchstat
76-
id: benchstat
126+
if: ${{ steps.baseline-restore.outputs.cache-hit != '' }}
77127
run: |
78-
go run golang.org/x/perf/cmd/benchstat@latest bench-main-prev.txt bench-main-curr.txt | tee -a $GITHUB_OUTPUT bench-stats.txt
128+
go run golang.org/x/perf/cmd/benchstat@latest bench-main-curr.txt bench-commit-*.txt | tee -a bench-stats.txt
79129
80130
CURR_VERSION="${GITHUB_SHA::7}"
81-
CURR_URL="https://github.com/mccutchen/websocket/commit/$CURR_VERSION"
82-
CURR_LINK="[$CURR_VERSION]($CURR_URL)"
83-
84-
PREV_VERSION="$(cat bench-version.txt 2>/dev/null)"
85-
PREV_URL="https://github.com/mccutchen/websocket/commit/$PREV_VERSION"
86-
PREV_LINK="[$PREV_VERSION]($PREV_URL)"
87-
88-
echo "### benchstats: $PREV_LINK (old) vs $CURR_LINK (new)" >>$GITHUB_STEP_SUMMARY
89-
echo '```' >>$GITHUB_STEP_SUMMARY
90-
cat bench-stats.txt >>$GITHUB_STEP_SUMMARY
91-
echo '```' >>$GITHUB_STEP_SUMMARY
92-
93-
- name: save new baseline results
94-
id: baseline-save
95-
uses: actions/cache/save@v4
131+
PREV_VERSION="$(cat bench-version-prev.txt 2>/dev/null)"
132+
COMPARE_URL="https://github.com/mccutchen/websocket/compare/$PREV_VERSION...$CURR_VERSION"
133+
COMPARE_LINK="[$PREV_VERSION...$CURR_VERSION]($COMPARE_URL)"
134+
135+
echo "### benchstats: $COMPARE_LINK" >>pr_comment
136+
echo '```' >>pr_comment
137+
cat bench-stats.txt >>pr_comment
138+
echo '```' >>pr_comment
139+
140+
echo "XXX FINAL PR COMMENT:"
141+
echo "================================================================================"
142+
cat pr_comment
143+
echo "================================================================================"
144+
145+
- name: post benchmark results
146+
uses: marocchino/sticky-pull-request-comment@v2
96147
with:
97-
path: |
98-
bench-*.txt
99-
key: ${{ runner.os }}-bench-main
148+
path: pr_comment

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
COVERAGE_PATH ?= coverage.out
33
COVERAGE_ARGS ?= -covermode=atomic -coverprofile=$(COVERAGE_PATH)
44
TEST_ARGS ?= -race -timeout 60s -count=1
5-
BENCH_COUNT ?= 5
5+
BENCH_COUNT ?= 10
66
BENCH_ARGS ?= -bench=. -benchmem -count=$(BENCH_COUNT)
77

88
# 3rd party tools
@@ -56,7 +56,7 @@ testautobahn:
5656
bench:
5757
go test $(BENCH_ARGS)
5858
.PHONY: bench
59-
59+
6060
lint:
6161
test -z "$$($(CMD_GOFUMPT) -d -e .)" || (echo "Error: gofmt failed"; $(CMD_GOFUMPT) -d -e . ; exit 1)
6262
go vet ./...

‎websocket_benchmark_test.go‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ func makeFrame(opcode websocket.Opcode, fin bool, payloadLen int) *websocket.Fra
2828

2929
func BenchmarkReadFrame(b *testing.B) {
3030
frameSizes := []int{
31-
256,
31+
// 256,
3232
1024,
33-
256 * 1024,
33+
// 256 * 1024,
3434
1024 * 1024,
3535
// largest cases from the autobahn test suite
3636
8 * 1024 * 1024,
@@ -64,13 +64,19 @@ func BenchmarkReadMessage(b *testing.B) {
6464
msgSize int
6565
frameCount int
6666
}{
67-
{16 * 1024, 4},
68-
{16 * 1024, 16},
69-
{1024 * 1024, 4},
70-
// worst case sizes from autobahn test suite
67+
// 1 frame per message
68+
{1024 * 1024, 1},
7169
{8 * 1024 * 1024, 1},
72-
{8 * 1024 * 1024, 8},
7370
{16 * 1024 * 1024, 1},
71+
72+
// 4 frames per message
73+
{1024 * 1024, 4},
74+
{8 * 1024 * 1024, 4},
75+
{16 * 1024 * 1024, 4},
76+
77+
// 16 frames per message
78+
{1024 * 1024, 16},
79+
{8 * 1024 * 1024, 16},
7480
{16 * 1024 * 1024, 16},
7581
}
7682

0 commit comments

Comments
 (0)