From 73d0bd24371fa1efdf45f13104bd7ba0efaec34b Mon Sep 17 00:00:00 2001 From: Lee Kellogg Date: Tue, 3 Mar 2026 14:42:43 -0500 Subject: [PATCH 1/3] Fix apptesting:execute The test case was not being converted to use the success_criteria field before including it in the request --- src/appdistribution/yaml_helper.ts | 26 ++++++++++++++------------ src/apptesting/parseTestFiles.ts | 3 ++- src/apptesting/types.ts | 2 +- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts index caedb886717..273b314da7b 100644 --- a/src/appdistribution/yaml_helper.ts +++ b/src/appdistribution/yaml_helper.ts @@ -1,8 +1,8 @@ import * as jsYaml from "js-yaml"; import { getErrMsg, FirebaseError } from "../error"; -import { TestCase } from "./types"; +import { TestCase, AiStep } from "./types"; -declare interface YamlStep { +export declare interface YamlStep { goal?: string; hint?: string; finalScreenAssertion?: string; @@ -64,22 +64,24 @@ function checkAllowedKeys(allowedKeys: Set, o: object) { } } +export function fromYamlStep(yamlStep: YamlStep): AiStep { + checkAllowedKeys(ALLOWED_YAML_STEP_KEYS, yamlStep); + return { + goal: castExists(yamlStep.goal, "goal"), + ...(yamlStep.hint && { hint: yamlStep.hint }), + ...(yamlStep.finalScreenAssertion && { + successCriteria: yamlStep.finalScreenAssertion, + }), + }; +} + function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): TestCase[] { return yamlTestCases.map((yamlTestCase) => { checkAllowedKeys(ALLOWED_YAML_TEST_CASE_KEYS, yamlTestCase); return { displayName: castExists(yamlTestCase.displayName, "displayName"), aiInstructions: { - steps: castExists(yamlTestCase.steps, "steps").map((yamlStep) => { - checkAllowedKeys(ALLOWED_YAML_STEP_KEYS, yamlStep); - return { - goal: castExists(yamlStep.goal, "goal"), - ...(yamlStep.hint && { hint: yamlStep.hint }), - ...(yamlStep.finalScreenAssertion && { - successCriteria: yamlStep.finalScreenAssertion, - }), - }; - }), + steps: castExists(yamlTestCase.steps, "steps").map((yamlStep) => fromYamlStep(yamlStep)), }, ...(yamlTestCase.id && { name: `${appName}/testCases/${yamlTestCase.id}`, diff --git a/src/apptesting/parseTestFiles.ts b/src/apptesting/parseTestFiles.ts index ba0e94060b0..4f4e1ee1b4d 100644 --- a/src/apptesting/parseTestFiles.ts +++ b/src/apptesting/parseTestFiles.ts @@ -4,6 +4,7 @@ import { logger } from "../logger"; import { Browser, TestCaseInvocation, TestStep } from "./types"; import { readFileFromDirectory, wrappedSafeLoad } from "../utils"; import { FirebaseError, getErrMsg, getError } from "../error"; +import { fromYamlStep, YamlStep } from "../appdistribution/yaml_helper"; export async function parseTestFiles( dir: string, @@ -143,7 +144,7 @@ function toTestCaseInvocation( prerequisiteTestCaseId: testDef.prerequisiteTestCaseId, startUri: targetUri + route, displayName: testDef.displayName, - steps: steps, + steps: steps.map((step: YamlStep) => fromYamlStep(step)), }, testExecution: browsers.map((browser) => ({ config: { browser } })), }; diff --git a/src/apptesting/types.ts b/src/apptesting/types.ts index a6c6f2d7139..e0fbe8d618b 100644 --- a/src/apptesting/types.ts +++ b/src/apptesting/types.ts @@ -1,6 +1,6 @@ export interface TestStep { goal: string; - finalScreenAssertion?: string; + successCriteria?: string; hint?: string; } From 5f07c84a99356bf9f41e303a793d6aba07153a26 Mon Sep 17 00:00:00 2001 From: Lee Kellogg Date: Tue, 3 Mar 2026 15:47:08 -0500 Subject: [PATCH 2/3] Update tests --- src/apptesting/invokeTests.spec.ts | 4 ++-- src/apptesting/parseTestFiles.spec.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/apptesting/invokeTests.spec.ts b/src/apptesting/invokeTests.spec.ts index 6ba1e3dbddb..55027bc3e8e 100644 --- a/src/apptesting/invokeTests.spec.ts +++ b/src/apptesting/invokeTests.spec.ts @@ -55,7 +55,7 @@ describe("invokeTests", () => { testCase: { startUri: "https://www.example.com", displayName: "testName2", - steps: [{ goal: "retest it", finalScreenAssertion: "a dialog appears" }], + steps: [{ goal: "retest it", successCriteria: "a dialog appears" }], }, testExecution: [{ config: { browser: Browser.CHROME } }], }, @@ -89,7 +89,7 @@ describe("invokeTests", () => { steps: [ { goal: "retest it", - finalScreenAssertion: "a dialog appears", + successCriteria: "a dialog appears", }, ], startUri: "https://www.example.com", diff --git a/src/apptesting/parseTestFiles.spec.ts b/src/apptesting/parseTestFiles.spec.ts index 494c4aa96c4..f9022c24e3d 100644 --- a/src/apptesting/parseTestFiles.spec.ts +++ b/src/apptesting/parseTestFiles.spec.ts @@ -82,7 +82,7 @@ describe("parseTestFiles", () => { { goal: "View the provided application", hint: "No additional actions should be necessary", - finalScreenAssertion: "The application should load with no obvious errors", + successCriteria: "The application should load with no obvious errors", }, ], }, From 094a6ed603baa4f62393df4f930998fde6f421c0 Mon Sep 17 00:00:00 2001 From: Lee Kellogg Date: Tue, 3 Mar 2026 15:57:43 -0500 Subject: [PATCH 3/3] Add javadoc --- src/appdistribution/yaml_helper.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts index 273b314da7b..2309a37c5d8 100644 --- a/src/appdistribution/yaml_helper.ts +++ b/src/appdistribution/yaml_helper.ts @@ -2,6 +2,7 @@ import * as jsYaml from "js-yaml"; import { getErrMsg, FirebaseError } from "../error"; import { TestCase, AiStep } from "./types"; +/** An AI test step in YAML format. */ export declare interface YamlStep { goal?: string; hint?: string; @@ -45,6 +46,7 @@ function toYamlTestCases(testCases: TestCase[]): YamlTestCase[] { })); } +/** Converts a list of test cases to YAML format. */ export function toYaml(testCases: TestCase[]): string { return jsYaml.safeDump({ tests: toYamlTestCases(testCases) }); } @@ -64,6 +66,7 @@ function checkAllowedKeys(allowedKeys: Set, o: object) { } } +/** Converts an AI test step from YAML format. */ export function fromYamlStep(yamlStep: YamlStep): AiStep { checkAllowedKeys(ALLOWED_YAML_STEP_KEYS, yamlStep); return { @@ -93,6 +96,7 @@ function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): Test }); } +/** Converts a list of test cases from YAML format. */ export function fromYaml(appName: string, yaml: string): TestCase[] { let parsedYaml: unknown; try {