Skip to content

chore: mixed benchmark test #3223

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chatton merged 17 commits into from
Apr 1, 2026
Merged

chore: mixed benchmark test #3223

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
23d0718
feat(benchmarking): add structured result output via BENCH_RESULT_OUTPUT
chatton Mar 24, 2026
3d8a60a
fix: address PR review feedback for structured benchmark output
chatton Mar 24, 2026
3dc3e35
Merge branch 'main' into cian/benchmark-results-artifact
chatton Mar 24, 2026
c0592ee
refactor: make spamoor config fully configurable via BENCH_* env vars
chatton Mar 24, 2026
a941a30
feat: extract host metadata from OTEL resource attributes in trace spans
chatton Mar 24, 2026
c55bec1
fix: defer emitRunResult so results are written even on test failure
chatton Mar 24, 2026
2bab161
fix: state pressure benchmark CI failure and align with other tests
chatton Mar 24, 2026
bab7a5f
fix: use deployment-level service names for trace queries in external…
chatton Mar 25, 2026
f2936c8
perf: use limit=1 for resource attribute trace queries
chatton Mar 25, 2026
1028b10
Merge branch 'main' into cian/benchmark-results-artifact
chatton Mar 25, 2026
41a4e40
Merge branch 'main' into cian/benchmark-results-artifact
chatton Mar 26, 2026
3fb75ee
docs: add missing doc comments to run_result.go functions
chatton Mar 26, 2026
f0eb3a7
Merge branch 'main' into cian/benchmark-results-artifact
chatton Mar 26, 2026
f0f71bd
Merge branch 'main' into cian/benchmark-results-artifact
chatton Mar 30, 2026
09f1b19
feat: adding mixed benchmarking test
chatton Mar 30, 2026
e5073ee
chore: merge main
chatton Apr 1, 2026
8410d69
fix: address review feedback on mixed benchmark test
chatton Apr 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/grpc/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ require (
github.com/googleapis/gax-go/v2 v2.19.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/hashicorp/go-hclog v1.6.2 // indirect
github.com/hashicorp/go-hclog v1.6.3 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-metrics v0.5.4 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
Expand Down
3 changes: 2 additions & 1 deletion apps/grpc/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -650,8 +650,9 @@ github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyN
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I=
github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
Expand Down
4 changes: 2 additions & 2 deletions execution/evm/test/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,8 +390,8 @@ github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NM
github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE=
github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I=
github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ require (
github.com/googleapis/gax-go/v2 v2.19.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/hashicorp/go-hclog v1.6.2 // indirect
github.com/hashicorp/go-hclog v1.6.3 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-metrics v0.5.4 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
Expand Down
3 changes: 2 additions & 1 deletion go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,8 +331,9 @@ github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyN
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I=
github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/benchmark/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func newBenchConfig(serviceName string) benchConfig {
SlotDuration: envOrDefault("BENCH_SLOT_DURATION", "250ms"),
GasLimit: envOrDefault("BENCH_GAS_LIMIT", ""),
ScrapeInterval: envOrDefault("BENCH_SCRAPE_INTERVAL", "1s"),
NumSpammers: envInt("BENCH_NUM_SPAMMERS", 2),
NumSpammers: envInt("BENCH_NUM_SPAMMERS", 4),
CountPerSpammer: envInt("BENCH_COUNT_PER_SPAMMER", 5000),
Throughput: envInt("BENCH_THROUGHPUT", 200),
WarmupTxs: envInt("BENCH_WARMUP_TXS", 200),
Expand Down
78 changes: 78 additions & 0 deletions test/e2e/benchmark/distribute_spammers_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
//go:build evm

package benchmark

import (
"testing"

"github.com/stretchr/testify/require"
)

func TestDistributeSpammers(t *testing.T) {
tests := []struct {
name string
total int
pcts [4]int
expected [4]int
}{
{
name: "minimum total equals types",
total: 4,
pcts: [4]int{40, 30, 20, 10},
expected: [4]int{1, 1, 1, 1},
},
{
name: "equal percentages",
total: 8,
pcts: [4]int{25, 25, 25, 25},
expected: [4]int{2, 2, 2, 2},
},
{
name: "default mix 8 spammers",
total: 8,
pcts: [4]int{40, 30, 20, 10},
expected: [4]int{3, 2, 2, 1},
},
{
name: "large total distributes proportionally",
total: 104,
pcts: [4]int{40, 30, 20, 10},
expected: [4]int{41, 31, 21, 11},
},
{
name: "sum of counts equals total",
total: 7,
pcts: [4]int{40, 30, 20, 10},
expected: [4]int{2, 2, 2, 1},
},
{
name: "5 spammers with uneven split",
total: 5,
pcts: [4]int{40, 30, 20, 10},
expected: [4]int{2, 1, 1, 1},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := distributeSpammers(tt.total, tt.pcts)
require.Equal(t, tt.expected, result)

sum := result[0] + result[1] + result[2] + result[3]
require.Equal(t, tt.total, sum, "sum of distributed spammers must equal total")

for i, c := range result {
require.GreaterOrEqual(t, c, 1, "type %d must have at least 1 spammer", i)
}
})
}
}

func TestDistributeSpammersPanicsOnLowTotal(t *testing.T) {
require.Panics(t, func() {
distributeSpammers(3, [4]int{40, 30, 20, 10})
})
require.Panics(t, func() {
distributeSpammers(0, [4]int{25, 25, 25, 25})
})
}
6 changes: 2 additions & 4 deletions test/e2e/benchmark/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -471,11 +471,9 @@ func waitForMetricTarget(t testing.TB, name string, poll func() (float64, error)
}
select {
case <-ctx.Done():
t.Logf("metric %s: context cancelled (target %.0f)", name, target)
return
t.Fatalf("metric %s: context cancelled (target %.0f)", name, target)
case <-timer.C:
t.Logf("metric %s did not reach target %.0f within %v", name, target, timeout)
return
t.Fatalf("metric %s did not reach target %.0f within %v", name, target, timeout)
case <-ticker.C:
}
}
Expand Down
Loading
Loading