Skip to content

Commit be1a9c8

Browse files
dnephindnephin
committed
Add reun-fails-report
This report can be used in a CI job to report back to the PR which tests flaked so that flaky tests are still easily visible.
1 parent 9a2c7b4 commit be1a9c8

File tree

8 files changed

+172
-2
lines changed

8 files changed

+172
-2
lines changed

‎.golangci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
linters-settings:
22
gocyclo:
3-
min-complexity: 10
3+
min-complexity: 12
44
goconst:
55
min-len: 2
66
min-occurrences: 4

‎main.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
107107
"do not rerun any tests if the initial run has more than this number of failures")
108108
flags.Var((*stringSlice)(&opts.packages), "packages",
109109
"space separated list of package to test")
110+
flags.StringVar(&opts.rerunFailsReportFile, "rerun-fails-report", "",
111+
"write a report to the file, of the tests that were rerun")
110112

111113
flags.BoolVar(&opts.debug, "debug", false, "enabled debug logging")
112114
flags.BoolVar(&opts.version, "version", false, "show version and exit")
@@ -158,6 +160,7 @@ type options struct {
158160
junitTestCaseClassnameFormat *junitFieldFormatValue
159161
rerunFailsMaxAttempts int
160162
rerunFailsMaxInitialFailures int
163+
rerunFailsReportFile string
161164
packages []string
162165
version bool
163166

@@ -223,6 +226,9 @@ func run(opts *options) error {
223226
if err := writeJUnitFile(opts, exec); err != nil {
224227
return err
225228
}
229+
if err := writeRerunFailsReport(opts, exec); err != nil {
230+
return err
231+
}
226232
if err := postRunHook(opts, exec); err != nil {
227233
return err
228234
}

‎rerunfails.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package main
33
import (
44
"context"
55
"fmt"
6+
"os"
7+
"sort"
68
"strings"
79

810
"github.com/pkg/errors"
@@ -129,3 +131,53 @@ func goTestRunFlagFromTestCases(tcs []string) string {
129131
buf.WriteString(")$")
130132
return buf.String()
131133
}
134+
135+
func writeRerunFailsReport(opts *options, exec *testjson.Execution) error {
136+
if opts.rerunFailsMaxAttempts == 0 || opts.rerunFailsReportFile == "" {
137+
return nil
138+
}
139+
140+
type testCaseCounts struct {
141+
total int
142+
failed int
143+
}
144+
145+
names := []string{}
146+
results := map[string]testCaseCounts{}
147+
for _, failure := range exec.Failed() {
148+
name := failure.Package + "." + failure.Test
149+
if _, ok := results[name]; ok {
150+
continue
151+
}
152+
names = append(names, name)
153+
154+
pkg := exec.Package(failure.Package)
155+
counts := testCaseCounts{}
156+
157+
for _, tc := range pkg.Failed {
158+
if tc.Test == failure.Test {
159+
counts.total++
160+
counts.failed++
161+
}
162+
}
163+
for _, tc := range pkg.Passed {
164+
if tc.Test == failure.Test {
165+
counts.total++
166+
}
167+
}
168+
// Skipped tests are not counted, but presumably skipped tests can not fail
169+
results[name] = counts
170+
}
171+
172+
fh, err := os.Create(opts.rerunFailsReportFile)
173+
if err != nil {
174+
return err
175+
}
176+
177+
sort.Strings(names)
178+
for _, name := range names {
179+
counts := results[name]
180+
fmt.Fprintf(fh, "%s: %d runs, %d failures\n", name, counts.total, counts.failed)
181+
}
182+
return nil
183+
}

‎rerunfails_test.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"io/ioutil"
6+
"testing"
7+
8+
"gotest.tools/gotestsum/testjson"
9+
"gotest.tools/v3/assert"
10+
"gotest.tools/v3/fs"
11+
"gotest.tools/v3/golden"
12+
)
13+
14+
func TestWriteRerunFailsReport(t *testing.T) {
15+
reportFile := fs.NewFile(t, t.Name())
16+
defer reportFile.Remove()
17+
18+
opts := &options{
19+
rerunFailsReportFile: reportFile.Path(),
20+
rerunFailsMaxAttempts: 4,
21+
}
22+
23+
exec, err := testjson.ScanTestOutput(testjson.ScanConfig{
24+
Stdout: bytes.NewReader(golden.Get(t, "go-test-json-flaky-rerun.out")),
25+
})
26+
assert.NilError(t, err)
27+
28+
err = writeRerunFailsReport(opts, exec)
29+
assert.NilError(t, err)
30+
31+
raw, err := ioutil.ReadFile(reportFile.Path())
32+
assert.NilError(t, err)
33+
golden.Assert(t, string(raw), t.Name()+"-expected")
34+
}

‎testdata/TestWriteRerunFailsReport-expected‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
gotest.tools/gotestsum/testdata/e2e/flaky.TestFailsOften: 4 runs, 3 failures
2+
gotest.tools/gotestsum/testdata/e2e/flaky.TestFailsRarely: 2 runs, 1 failures
3+
gotest.tools/gotestsum/testdata/e2e/flaky.TestFailsSometimes: 3 runs, 2 failures

‎testdata/go-test-json-flaky-rerun.out‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
{"Time":"2020-06-21T21:12:10.815884042-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses"}
2+
{"Time":"2020-06-21T21:12:10.816009964-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses","Output":"=== RUN TestAlwaysPasses\n"}
3+
{"Time":"2020-06-21T21:12:10.816024167-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses","Output":"--- PASS: TestAlwaysPasses (0.00s)\n"}
4+
{"Time":"2020-06-21T21:12:10.81603019-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses","Elapsed":0}
5+
{"Time":"2020-06-21T21:12:10.81604135-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely"}
6+
{"Time":"2020-06-21T21:12:10.816045457-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"=== RUN TestFailsRarely\n"}
7+
{"Time":"2020-06-21T21:12:10.816049373-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"SEED: 0\n"}
8+
{"Time":"2020-06-21T21:12:10.816063218-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":" TestFailsRarely: flaky_test.go:51: not this time\n"}
9+
{"Time":"2020-06-21T21:12:10.816068373-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"--- FAIL: TestFailsRarely (0.00s)\n"}
10+
{"Time":"2020-06-21T21:12:10.816072877-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Elapsed":0}
11+
{"Time":"2020-06-21T21:12:10.81607599-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes"}
12+
{"Time":"2020-06-21T21:12:10.81607897-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"=== RUN TestFailsSometimes\n"}
13+
{"Time":"2020-06-21T21:12:10.816082575-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"SEED: 0\n"}
14+
{"Time":"2020-06-21T21:12:10.816086165-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":" TestFailsSometimes: flaky_test.go:58: not this time\n"}
15+
{"Time":"2020-06-21T21:12:10.8160903-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"--- FAIL: TestFailsSometimes (0.00s)\n"}
16+
{"Time":"2020-06-21T21:12:10.816094667-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Elapsed":0}
17+
{"Time":"2020-06-21T21:12:10.816098149-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
18+
{"Time":"2020-06-21T21:12:10.816101281-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN TestFailsOften\n"}
19+
{"Time":"2020-06-21T21:12:10.81610482-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED: 0\n"}
20+
{"Time":"2020-06-21T21:12:10.816108597-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":" TestFailsOften: flaky_test.go:65: not this time\n"}
21+
{"Time":"2020-06-21T21:12:10.816112698-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- FAIL: TestFailsOften (0.00s)\n"}
22+
{"Time":"2020-06-21T21:12:10.816116235-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
23+
{"Time":"2020-06-21T21:12:10.816119536-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch"}
24+
{"Time":"2020-06-21T21:12:10.816122935-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch","Output":"=== RUN TestFailsOftenDoesNotPrefixMatch\n"}
25+
{"Time":"2020-06-21T21:12:10.816127081-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch","Output":"--- PASS: TestFailsOftenDoesNotPrefixMatch (0.00s)\n"}
26+
{"Time":"2020-06-21T21:12:10.816131578-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch","Elapsed":0}
27+
{"Time":"2020-06-21T21:12:10.816135262-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch"}
28+
{"Time":"2020-06-21T21:12:10.816139844-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch","Output":"=== RUN TestFailsSometimesDoesNotPrefixMatch\n"}
29+
{"Time":"2020-06-21T21:12:10.816149774-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch","Output":"--- PASS: TestFailsSometimesDoesNotPrefixMatch (0.00s)\n"}
30+
{"Time":"2020-06-21T21:12:10.816155804-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch","Elapsed":0}
31+
{"Time":"2020-06-21T21:12:10.816159612-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\n"}
32+
{"Time":"2020-06-21T21:12:10.816211999-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\tgotest.tools/gotestsum/testdata/e2e/flaky\t0.001s\n"}
33+
{"Time":"2020-06-21T21:12:10.816224311-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0.001}
34+
{"Time":"2020-06-21T21:12:10.985779906-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely"}
35+
{"Time":"2020-06-21T21:12:10.985890459-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"=== RUN TestFailsRarely\n"}
36+
{"Time":"2020-06-21T21:12:10.985902826-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"SEED: 1\n"}
37+
{"Time":"2020-06-21T21:12:10.985911982-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"--- PASS: TestFailsRarely (0.00s)\n"}
38+
{"Time":"2020-06-21T21:12:10.985916034-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Elapsed":0}
39+
{"Time":"2020-06-21T21:12:10.985923087-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes"}
40+
{"Time":"2020-06-21T21:12:10.985926869-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"=== RUN TestFailsSometimes\n"}
41+
{"Time":"2020-06-21T21:12:10.985930857-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"SEED: 1\n"}
42+
{"Time":"2020-06-21T21:12:10.985934726-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":" TestFailsSometimes: flaky_test.go:58: not this time\n"}
43+
{"Time":"2020-06-21T21:12:10.985939499-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"--- FAIL: TestFailsSometimes (0.00s)\n"}
44+
{"Time":"2020-06-21T21:12:10.985943642-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Elapsed":0}
45+
{"Time":"2020-06-21T21:12:10.985945782-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
46+
{"Time":"2020-06-21T21:12:10.985947733-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN TestFailsOften\n"}
47+
{"Time":"2020-06-21T21:12:10.985949865-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED: 1\n"}
48+
{"Time":"2020-06-21T21:12:10.985952001-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":" TestFailsOften: flaky_test.go:65: not this time\n"}
49+
{"Time":"2020-06-21T21:12:10.985954479-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- FAIL: TestFailsOften (0.00s)\n"}
50+
{"Time":"2020-06-21T21:12:10.985956607-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
51+
{"Time":"2020-06-21T21:12:10.985958719-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\n"}
52+
{"Time":"2020-06-21T21:12:10.986048472-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\tgotest.tools/gotestsum/testdata/e2e/flaky\t0.001s\n"}
53+
{"Time":"2020-06-21T21:12:10.986068802-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0.001}
54+
{"Time":"2020-06-21T21:12:11.147554767-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes"}
55+
{"Time":"2020-06-21T21:12:11.147676606-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"=== RUN TestFailsSometimes\n"}
56+
{"Time":"2020-06-21T21:12:11.14768948-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"SEED: 2\n"}
57+
{"Time":"2020-06-21T21:12:11.147700275-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"--- PASS: TestFailsSometimes (0.00s)\n"}
58+
{"Time":"2020-06-21T21:12:11.147705201-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Elapsed":0}
59+
{"Time":"2020-06-21T21:12:11.147710442-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
60+
{"Time":"2020-06-21T21:12:11.147712827-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN TestFailsOften\n"}
61+
{"Time":"2020-06-21T21:12:11.147715318-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED: 2\n"}
62+
{"Time":"2020-06-21T21:12:11.147717797-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":" TestFailsOften: flaky_test.go:65: not this time\n"}
63+
{"Time":"2020-06-21T21:12:11.147721095-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- FAIL: TestFailsOften (0.00s)\n"}
64+
{"Time":"2020-06-21T21:12:11.147723329-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
65+
{"Time":"2020-06-21T21:12:11.147725838-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\n"}
66+
{"Time":"2020-06-21T21:12:11.14783256-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\tgotest.tools/gotestsum/testdata/e2e/flaky\t0.001s\n"}
67+
{"Time":"2020-06-21T21:12:11.147849384-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0.001}
68+
{"Time":"2020-06-21T21:12:11.226137617-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
69+
{"Time":"2020-06-21T21:12:11.226247164-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN TestFailsOften\n"}
70+
{"Time":"2020-06-21T21:12:11.226256816-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED: 14\n"}
71+
{"Time":"2020-06-21T21:12:11.226264297-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- PASS: TestFailsOften (0.00s)\n"}
72+
{"Time":"2020-06-21T21:12:11.226266905-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
73+
{"Time":"2020-06-21T21:12:11.22627268-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"PASS\n"}
74+
{"Time":"2020-06-21T21:12:11.226275849-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"ok \tgotest.tools/gotestsum/testdata/e2e/flaky\t(cached)\n"}
75+
{"Time":"2020-06-21T21:12:11.226279592-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0}

‎testdata/gotestsum-help-text‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ Flags:
1616
--raw-command don't prepend 'go test -json' to the 'go test' command
1717
--rerun-fails int[=2] rerun failed tests until they all pass, or attempts exceeds maximum. Defaults to max 2 reruns when enabled.
1818
--rerun-fails-max-failures int do not rerun any tests if the initial run has more than this number of failures (default 10)
19+
--rerun-fails-report string write a report to the file, of the tests that were rerun
1920
--version show version and exit
2021

2122
Formats:

‎testjson/execution.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,6 @@ func (e *Execution) addPackageEvent(pkg *Package, event TestEvent) {
285285
}
286286
}
287287

288-
// nolint: gocyclo
289288
func (p *Package) addTestEvent(event TestEvent) {
290289
tc := p.running[event.Test]
291290
root, subTest := splitTestName(event.Test)

0 commit comments

Comments
 (0)