Skip to content

Commit 27c370b

Browse files
bootstraponlinebootstraponline
committed
Use matrix rollup
1 parent a2fe405 commit 27c370b

8 files changed

Lines changed: 87 additions & 63 deletions

File tree

‎test_runner/src/main/kotlin/ftl/gc/GcToolResults.kt‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
package ftl.gc
22

3+
import com.google.api.services.testing.model.TestExecution
34
import com.google.api.services.testing.model.ToolResultsHistory
45
import com.google.api.services.testing.model.ToolResultsStep
56
import com.google.api.services.toolresults.ToolResults
7+
import com.google.api.services.toolresults.model.Execution
68
import com.google.api.services.toolresults.model.History
79
import com.google.api.services.toolresults.model.ListTestCasesResponse
810
import com.google.api.services.toolresults.model.Step
@@ -27,7 +29,7 @@ object GcToolResults {
2729

2830
// https://github.com/bootstraponline/gcloud_cli/blob/0752e88b155a417a18d244c242b4ab3fb9aa1c1f/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/history_picker.py#L45
2931
private fun listHistoriesByName(args: IArgs): List<History> {
30-
val result = GcToolResults.service
32+
val result = service
3133
.projects()
3234
.histories()
3335
.list(args.project)
@@ -40,7 +42,7 @@ object GcToolResults {
4042
val history = History()
4143
.setName(args.resultsHistoryName)
4244
.setDisplayName(args.resultsHistoryName)
43-
return GcToolResults.service
45+
return service
4446
.projects()
4547
.histories()
4648
.create(args.project, history)
@@ -61,8 +63,28 @@ object GcToolResults {
6163
return createHistory(args).historyId
6264
}
6365

64-
fun getResults(toolResultsStep: ToolResultsStep): Step {
65-
return GcToolResults.service
66+
// FetchMatrixRollupOutcome - https://github.com/bootstraponline/gcloud_cli/blob/137d864acd5928baf25434cf59b0225c4d1f9319/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/results_summary.py#L122
67+
// Get the rolled-up outcome for a test execution from the Tool Results service.
68+
// Prefer execution rollUp outcome over individual step outcome
69+
// https://github.com/bootstraponline/gcloud_cli/blob/137d864acd5928baf25434cf59b0225c4d1f9319/google-cloud-sdk/lib/googlecloudsdk/third_party/apis/toolresults_v1beta3.json#L992
70+
fun getExecutionResult(testExecution: TestExecution): Execution {
71+
val toolResultsStep = testExecution.toolResultsStep
72+
73+
// Toolresults.Projects.Histories.Executions.GetRequest
74+
return service
75+
.projects()
76+
.histories()
77+
.executions()
78+
.get(
79+
toolResultsStep.projectId,
80+
toolResultsStep.historyId,
81+
toolResultsStep.executionId
82+
)
83+
.executeWithRetry()
84+
}
85+
86+
fun getStepResult(toolResultsStep: ToolResultsStep): Step {
87+
return service
6688
.projects()
6789
.histories()
6890
.executions()
@@ -78,7 +100,7 @@ object GcToolResults {
78100

79101
// Lists Test Cases attached to a Step
80102
fun listTestCases(toolResultsStep: ToolResultsStep): ListTestCasesResponse {
81-
return GcToolResults.service
103+
return service
82104
.projects()
83105
.histories()
84106
.executions()
@@ -93,7 +115,7 @@ object GcToolResults {
93115
}
94116

95117
fun getDefaultBucket(projectId: String): String? {
96-
val response = GcToolResults.service.Projects().initializeSettings(projectId).executeWithRetry()
118+
val response = service.Projects().initializeSettings(projectId).executeWithRetry()
97119
return response.defaultBucket
98120
}
99121
}

‎test_runner/src/main/kotlin/ftl/json/MatrixMap.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,6 @@ class MatrixMap(
5050
}
5151

5252
private fun SavedMatrix.logError(message: String) {
53-
println("Error: Matrix $message: ${this.matrixId} ${this.state} ${this.webLink}")
53+
println("Error: Matrix $message: ${this.matrixId} ${this.state} ${this.outcome} ${this.outcomeDetails} ${this.webLink}")
5454
}
5555
}

‎test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,12 +75,13 @@ class SavedMatrix(matrix: TestMatrix) {
7575
if (matrix.testExecutions == null) return
7676

7777
matrix.testExecutions.forEach {
78-
val step = GcToolResults.getResults(it.toolResultsStep)
79-
updateOutcome(step.outcome)
78+
val executionResult = GcToolResults.getExecutionResult(it)
79+
updateOutcome(executionResult.outcome)
8080

8181
// testExecutionStep, testTiming, etc. can all be null.
8282
// sometimes testExecutionStep is present and testTiming is null
83-
val testTimeSeconds = step.testExecutionStep?.testTiming?.testProcessDuration?.seconds ?: return
83+
val stepResult = GcToolResults.getStepResult(it.toolResultsStep)
84+
val testTimeSeconds = stepResult.testExecutionStep?.testTiming?.testProcessDuration?.seconds ?: return
8485

8586
val billableMinutes = Billing.billableMinutes(testTimeSeconds)
8687

‎test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package ftl.reports
33
import ftl.args.IArgs
44
import ftl.config.FtlConstants.indent
55
import ftl.json.MatrixMap
6-
import ftl.json.SavedMatrix
76
import ftl.reports.util.IReport
87
import ftl.reports.xml.model.JUnitTestResult
98
import ftl.util.Utils.println
@@ -29,13 +28,11 @@ object MatrixResultsReport : IReport {
2928
private fun generate(matrices: MatrixMap): String {
3029
var total = 0
3130
var success = 0
32-
val failedMatrices = mutableListOf<SavedMatrix>()
3331
matrices.map.values.forEach { matrix ->
3432
total += 1
3533

36-
if (matrix.failed()) {
37-
failedMatrices.add(matrix)
38-
} else {
34+
// unfinished matrix will not be reported as failed since it's still running
35+
if (matrix.failed().not()) {
3936
success += 1
4037
}
4138
}
@@ -52,11 +49,6 @@ object MatrixResultsReport : IReport {
5249
if (failed > 0) {
5350
writer.println("$indent$failed matrices failed")
5451
writer.println()
55-
failedMatrices.forEach {
56-
writer.println("$indent${it.matrixId} ${it.outcomeDetails}")
57-
writer.println("$indent${it.webLink}")
58-
writer.println()
59-
}
6052
}
6153

6254
return writer.toString()

‎test_runner/src/main/kotlin/ftl/util/StepOutcome.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package ftl.util
22

33
// ToolResults API step outcome values
44
object StepOutcome {
5-
// https://github.com/bootstraponline/gcloud_cli/blob/master/google-cloud-sdk/lib/googlecloudsdk/third_party/apis/toolresults_v1beta3.json#L755
5+
// https://github.com/bootstraponline/gcloud_cli/blob/master/google-cloud-sdk/lib/googlecloudsdk/third_party/apis/toolresults_v1beta3.json#L610
66
const val failure = "failure"
77
const val flaky = "flaky"
88
const val inconclusive = "inconclusive"

‎test_runner/src/test/kotlin/Tmp.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ object Tmp {
8787
val toolResult = gson.fromJson(content, ToolResultsStep::class.java)
8888

8989
val tests = GcToolResults.listTestCases(toolResult)
90-
val result = GcToolResults.getResults(toolResult)
90+
val result = GcToolResults.getStepResult(toolResult)
9191

9292
// todo: handle multiple overviews (iOS only)
9393
val overview = result.testExecutionStep.testSuiteOverviews.first()

‎test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ class SavedMatrixTest {
2525
val toolResultsStep = ToolResultsStep()
2626
toolResultsStep.projectId = "1"
2727
toolResultsStep.historyId = "2"
28-
toolResultsStep.executionId = "3"
28+
toolResultsStep.executionId = stepId.toString()
2929
toolResultsStep.stepId = stepId.toString()
3030

3131
val testExecution = TestExecution()
@@ -75,7 +75,7 @@ class SavedMatrixTest {
7575
assertThat(savedMatrix.matrixId).isEqualTo(matrixId)
7676
assertThat(savedMatrix.state).isEqualTo(matrixState)
7777
assertThat(savedMatrix.gcsPath).isEqualTo(mockGcsPath)
78-
assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/3")
78+
assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/-1")
7979
assertThat(savedMatrix.downloaded).isFalse()
8080
assertThat(savedMatrix.billableVirtualMinutes).isEqualTo(0)
8181
assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(2)
@@ -107,7 +107,7 @@ class SavedMatrixTest {
107107
assertThat(savedMatrix.matrixId).isEqualTo(matrixId)
108108
assertThat(savedMatrix.state).isEqualTo(matrixState)
109109
assertThat(savedMatrix.gcsPath).isEqualTo(mockGcsPath)
110-
assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/3/executions/-3")
110+
assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/null/testlab/histories/2/matrices/-3/executions/-3")
111111
assertThat(savedMatrix.downloaded).isFalse()
112112
assertThat(savedMatrix.billableVirtualMinutes).isEqualTo(0)
113113
assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(1)

‎test_runner/src/test/kotlin/ftl/test/util/MockServer.kt‎

Lines changed: 47 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,44 @@ object MockServer {
6767
private val androidCatalog by lazy { loadCatalog<AndroidDeviceCatalog>("android_catalog.json") }
6868
private val iosCatalog by lazy { loadCatalog<IosDeviceCatalog>("ios_catalog.json") }
6969

70+
private fun fakeStep(stringId: String): Step {
71+
val oneSecond = Duration().setSeconds(1)
72+
73+
val testTiming = TestTiming()
74+
.setTestProcessDuration(oneSecond)
75+
76+
val testExecutionStep = TestExecutionStep()
77+
.setTestTiming(testTiming)
78+
79+
val outcome = Outcome()
80+
when (stringId) {
81+
"-1" -> {
82+
outcome.summary = failure
83+
val failureDetail = FailureDetail()
84+
failureDetail.timedOut = true
85+
outcome.failureDetail = failureDetail
86+
}
87+
"-2" -> {
88+
outcome.summary = inconclusive
89+
val inconclusiveDetail = InconclusiveDetail()
90+
inconclusiveDetail.abortedByUser = true
91+
outcome.inconclusiveDetail = inconclusiveDetail
92+
}
93+
"-3" -> {
94+
outcome.summary = skipped
95+
val skippedDetail = SkippedDetail()
96+
skippedDetail.incompatibleAppVersion = true
97+
outcome.skippedDetail = skippedDetail
98+
}
99+
else -> outcome.summary = success
100+
}
101+
102+
return Step()
103+
.setTestExecutionStep(testExecutionStep)
104+
.setRunDuration(oneSecond)
105+
.setOutcome(outcome)
106+
}
107+
70108
val application by lazy {
71109
embeddedServer(Netty, port) {
72110
install(ContentNegotiation) {
@@ -144,49 +182,20 @@ object MockServer {
144182
call.respond(matrix)
145183
}
146184

147-
// GcToolResults.getResults(toolResultsStep)
185+
// GcToolResults.getStepResult(toolResultsStep)
148186
// GET /toolresults/v1beta3/projects/delta-essence-114723/histories/1/executions/1/steps/1
149187
get("/toolresults/v1beta3/projects/{project}/histories/{historyId}/executions/{executionId}/steps/{stepId}") {
150188
println("Responding to GET ${call.request.uri}")
151189
val stepId = call.parameters["stepId"] ?: ""
190+
call.respond(fakeStep(stepId))
191+
}
152192

153-
val oneSecond = Duration().setSeconds(1)
154-
155-
val testTiming = TestTiming()
156-
.setTestProcessDuration(oneSecond)
157-
158-
val testExecutionStep = TestExecutionStep()
159-
.setTestTiming(testTiming)
160-
161-
val outcome = Outcome()
162-
when (stepId) {
163-
"-1" -> {
164-
outcome.summary = failure
165-
val failureDetail = FailureDetail()
166-
failureDetail.timedOut = true
167-
outcome.failureDetail = failureDetail
168-
}
169-
"-2" -> {
170-
outcome.summary = inconclusive
171-
val inconclusiveDetail = InconclusiveDetail()
172-
inconclusiveDetail.abortedByUser = true
173-
outcome.inconclusiveDetail = inconclusiveDetail
174-
}
175-
"-3" -> {
176-
outcome.summary = skipped
177-
val skippedDetail = SkippedDetail()
178-
skippedDetail.incompatibleAppVersion = true
179-
outcome.skippedDetail = skippedDetail
180-
}
181-
else -> outcome.summary = success
182-
}
183-
184-
val step = Step()
185-
.setTestExecutionStep(testExecutionStep)
186-
.setRunDuration(oneSecond)
187-
.setOutcome(outcome)
188-
189-
call.respond(step)
193+
// GcToolResults.getExecutionResult(toolResultsStep)
194+
// GET /toolresults/v1beta3/projects/delta-essence-114723/histories/1/executions/1
195+
get("/toolresults/v1beta3/projects/{project}/histories/{historyId}/executions/{executionId}") {
196+
println("Responding to GET ${call.request.uri}")
197+
val executionId = call.parameters["executionId"] ?: ""
198+
call.respond(fakeStep(executionId))
190199
}
191200

192201
// GcToolResults.getDefaultBucket(project)

0 commit comments

Comments
 (0)