Skip to content

Commit 527a11a

Browse files
chattonchatton
authored
Script to dynamically generate list of e2e tests (E2E #1) (cosmos#1644)
1 parent 7d18182 commit 527a11a

File tree

4 files changed

+294
-2
lines changed

4 files changed

+294
-2
lines changed

‎.github/scripts/build_test_matrix.go‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"go/ast"
7+
"go/parser"
8+
"go/token"
9+
"io/fs"
10+
"os"
11+
"path/filepath"
12+
"strings"
13+
)
14+
15+
const (
16+
testNamePrefix = "Test"
17+
testFileNameSuffix = "_test.go"
18+
e2eTestDirectory = "e2e"
19+
)
20+
21+
// GithubActionTestMatrix represents
22+
type GithubActionTestMatrix struct {
23+
Include []TestSuitePair `json:"include"`
24+
}
25+
26+
type TestSuitePair struct {
27+
Test string `json:"test"`
28+
Suite string `json:"suite"`
29+
}
30+
31+
func main() {
32+
githubActionMatrix, err := getGithubActionMatrixForTests(e2eTestDirectory)
33+
if err != nil {
34+
fmt.Printf("error generating github action json: %s", err)
35+
os.Exit(1)
36+
}
37+
38+
ghBytes, err := json.Marshal(githubActionMatrix)
39+
if err != nil {
40+
fmt.Printf("error marshalling github action json: %s", err)
41+
os.Exit(1)
42+
}
43+
fmt.Println(string(ghBytes))
44+
}
45+
46+
// getGithubActionMatrixForTests returns a json string representing the contents that should go in the matrix
47+
// field in a github action workflow. This string can be used with `fromJSON(str)` to dynamically build
48+
// the workflow matrix to include all E2E tests under the e2eRootDirectory directory.
49+
func getGithubActionMatrixForTests(e2eRootDirectory string) (GithubActionTestMatrix, error) {
50+
testSuiteMapping := map[string][]string{}
51+
fset := token.NewFileSet()
52+
err := filepath.Walk(e2eRootDirectory, func(path string, info fs.FileInfo, err error) error {
53+
// only look at test files
54+
if !strings.HasSuffix(path, testFileNameSuffix) {
55+
return nil
56+
}
57+
58+
f, err := parser.ParseFile(fset, path, nil, 0)
59+
if err != nil {
60+
return fmt.Errorf("failed parsing file: %s", err)
61+
}
62+
63+
suiteNameForFile, testCases, err := extractSuiteAndTestNames(f)
64+
if err != nil {
65+
return fmt.Errorf("failed extracting test suite name and test cases: %s", err)
66+
}
67+
68+
testSuiteMapping[suiteNameForFile] = testCases
69+
70+
return nil
71+
})
72+
73+
if err != nil {
74+
return GithubActionTestMatrix{}, err
75+
}
76+
77+
gh := GithubActionTestMatrix{
78+
Include: []TestSuitePair{},
79+
}
80+
81+
for testSuiteName, testCases := range testSuiteMapping {
82+
for _, testCaseName := range testCases {
83+
gh.Include = append(gh.Include, TestSuitePair{
84+
Test: testCaseName,
85+
Suite: testSuiteName,
86+
})
87+
}
88+
}
89+
90+
return gh, nil
91+
}
92+
93+
// extractSuiteAndTestNames extracts the name of the test suite function as well
94+
// as all tests associated with it in the same file.
95+
func extractSuiteAndTestNames(file *ast.File) (string, []string, error) {
96+
var suiteNameForFile string
97+
var testCases []string
98+
99+
for _, d := range file.Decls {
100+
if f, ok := d.(*ast.FuncDecl); ok {
101+
functionName := f.Name.Name
102+
if isTestSuiteMethod(f) {
103+
if suiteNameForFile != "" {
104+
return "", nil, fmt.Errorf("found a second test function: %s when %s was already found", f.Name.Name, suiteNameForFile)
105+
}
106+
suiteNameForFile = functionName
107+
continue
108+
}
109+
if isTestFunction(f) {
110+
testCases = append(testCases, functionName)
111+
}
112+
}
113+
}
114+
if suiteNameForFile == "" {
115+
return "", nil, fmt.Errorf("file %s had no test suite test case", file.Name.Name)
116+
}
117+
return suiteNameForFile, testCases, nil
118+
}
119+
120+
// isTestSuiteMethod returns true if the function is a test suite function.
121+
// e.g. func TestFeeMiddlewareTestSuite(t *testing.T) { ... }
122+
func isTestSuiteMethod(f *ast.FuncDecl) bool {
123+
return strings.HasPrefix(f.Name.Name, testNamePrefix) && len(f.Type.Params.List) == 1
124+
}
125+
126+
// isTestFunction returns true if the function name starts with "Test" and has no parameters.
127+
// as test suite functions do not accept a *testing.T.
128+
func isTestFunction(f *ast.FuncDecl) bool {
129+
return strings.HasPrefix(f.Name.Name, testNamePrefix) && len(f.Type.Params.List) == 0
130+
}

‎.github/scripts/build_test_matrix_test.go‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
package main
2+
3+
import (
4+
"os"
5+
"path"
6+
"sort"
7+
"strings"
8+
"testing"
9+
10+
"github.com/stretchr/testify/assert"
11+
)
12+
13+
const (
14+
nonTestFile = "not_test_file.go"
15+
goTestFileNameOne = "first_go_file_test.go"
16+
goTestFileNameTwo = "second_go_file_test.go"
17+
)
18+
19+
func TestGetGithubActionMatrixForTests(t *testing.T) {
20+
t.Run("empty dir does not fail", func(t *testing.T) {
21+
testingDir := t.TempDir()
22+
_, err := getGithubActionMatrixForTests(testingDir)
23+
assert.NoError(t, err)
24+
})
25+
26+
t.Run("only test functions are picked up", func(t *testing.T) {
27+
testingDir := t.TempDir()
28+
createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, goTestFileNameOne)
29+
30+
gh, err := getGithubActionMatrixForTests(testingDir)
31+
assert.NoError(t, err)
32+
33+
expected := GithubActionTestMatrix{
34+
Include: []TestSuitePair{
35+
{
36+
Suite: "TestFeeMiddlewareTestSuite",
37+
Test: "TestA",
38+
},
39+
{
40+
Suite: "TestFeeMiddlewareTestSuite",
41+
Test: "TestB",
42+
},
43+
},
44+
}
45+
assertGithubActionTestMatricesEqual(t, expected, gh)
46+
})
47+
48+
t.Run("all files are picked up", func(t *testing.T) {
49+
testingDir := t.TempDir()
50+
createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, goTestFileNameOne)
51+
createFileWithTestSuiteAndTests(t, "TransferTestSuite", "TestC", "TestD", testingDir, goTestFileNameTwo)
52+
53+
gh, err := getGithubActionMatrixForTests(testingDir)
54+
assert.NoError(t, err)
55+
56+
expected := GithubActionTestMatrix{
57+
Include: []TestSuitePair{
58+
{
59+
Suite: "TestTransferTestSuite",
60+
Test: "TestC",
61+
},
62+
{
63+
Suite: "TestFeeMiddlewareTestSuite",
64+
Test: "TestA",
65+
},
66+
{
67+
Suite: "TestFeeMiddlewareTestSuite",
68+
Test: "TestB",
69+
},
70+
{
71+
Suite: "TestTransferTestSuite",
72+
Test: "TestD",
73+
},
74+
},
75+
}
76+
77+
assertGithubActionTestMatricesEqual(t, expected, gh)
78+
})
79+
80+
t.Run("non test files are not picked up", func(t *testing.T) {
81+
testingDir := t.TempDir()
82+
createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, nonTestFile)
83+
84+
gh, err := getGithubActionMatrixForTests(testingDir)
85+
assert.NoError(t, err)
86+
assert.Empty(t, gh.Include)
87+
})
88+
89+
t.Run("fails when there are multiple suite runs", func(t *testing.T) {
90+
testingDir := t.TempDir()
91+
createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, nonTestFile)
92+
93+
fileWithTwoSuites := `package foo
94+
func SuiteOne(t *testing.T) {
95+
suite.Run(t, new(FeeMiddlewareTestSuite))
96+
}
97+
98+
func SuiteTwo(t *testing.T) {
99+
suite.Run(t, new(FeeMiddlewareTestSuite))
100+
}
101+
102+
type FeeMiddlewareTestSuite struct {}
103+
`
104+
105+
err := os.WriteFile(path.Join(testingDir, goTestFileNameOne), []byte(fileWithTwoSuites), os.FileMode(777))
106+
assert.NoError(t, err)
107+
108+
_, err = getGithubActionMatrixForTests(testingDir)
109+
assert.Error(t, err)
110+
})
111+
}
112+
113+
func assertGithubActionTestMatricesEqual(t *testing.T, expected, actual GithubActionTestMatrix) {
114+
// sort by both suite and test as the order of the end result does not matter as
115+
// all tests will be run.
116+
sort.SliceStable(expected.Include, func(i, j int) bool {
117+
memberI := expected.Include[i]
118+
memberJ := expected.Include[j]
119+
if memberI.Suite == memberJ.Suite {
120+
return memberI.Test < memberJ.Test
121+
}
122+
return memberI.Suite < memberJ.Suite
123+
})
124+
125+
sort.SliceStable(actual.Include, func(i, j int) bool {
126+
memberI := actual.Include[i]
127+
memberJ := actual.Include[j]
128+
if memberI.Suite == memberJ.Suite {
129+
return memberI.Test < memberJ.Test
130+
}
131+
return memberI.Suite < memberJ.Suite
132+
})
133+
assert.Equal(t, expected.Include, actual.Include)
134+
}
135+
136+
func goTestFileContents(suiteName, fnName1, fnName2 string) string {
137+
138+
replacedSuiteName := strings.ReplaceAll(`package foo
139+
140+
func TestSuiteName(t *testing.T) {
141+
suite.Run(t, new(SuiteName))
142+
}
143+
144+
type SuiteName struct {}
145+
146+
func (s *SuiteName) fnName1() {}
147+
func (s *SuiteName) fnName2() {}
148+
149+
func (s *SuiteName) suiteHelper() {}
150+
151+
func helper() {}
152+
`, "SuiteName", suiteName)
153+
154+
replacedFn1Name := strings.ReplaceAll(replacedSuiteName, "fnName1", fnName1)
155+
return strings.ReplaceAll(replacedFn1Name, "fnName2", fnName2)
156+
}
157+
158+
func createFileWithTestSuiteAndTests(t *testing.T, suiteName, fn1Name, fn2Name, dir, filename string) {
159+
goFileContents := goTestFileContents(suiteName, fn1Name, fn2Name)
160+
err := os.WriteFile(path.Join(dir, filename), []byte(goFileContents), os.FileMode(777))
161+
assert.NoError(t, err)
162+
}

‎.github/workflows/test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ jobs:
6363
steps:
6464
- uses: actions/checkout@v3
6565
- name: Create a file with all the pkgs
66-
run: go list ./... > pkgs.txt
66+
run: go list ./... ./.github/scripts > pkgs.txt
6767
- name: Split pkgs into 4 files
6868
run: split -d -n l/4 pkgs.txt pkgs.txt.part.
6969
# cache multiple

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ view-docs:
216216
test: test-unit
217217
test-all: test-unit test-ledger-mock test-race test-cover
218218

219-
TEST_PACKAGES=./...
219+
TEST_PACKAGES=./... ./.github/scripts
220220
TEST_TARGETS := test-unit test-unit-amino test-unit-proto test-ledger-mock test-race test-ledger test-race
221221

222222
# Test runs-specific rules. To add a new test target, just add

0 commit comments

Comments
 (0)