From ad70fe305df344ed36634d82a4f994a36081e375 Mon Sep 17 00:00:00 2001 From: Lee Kellogg Date: Mon, 9 Feb 2026 18:20:32 -0500 Subject: [PATCH 1/5] Update yaml format for appdistribution:testcases:import/export --- src/appdistribution/types.ts | 2 +- src/appdistribution/yaml_helper.spec.ts | 137 ++++++++++++++---------- src/appdistribution/yaml_helper.ts | 28 +++-- 3 files changed, 99 insertions(+), 68 deletions(-) diff --git a/src/appdistribution/types.ts b/src/appdistribution/types.ts index e0e5e087832..dcfd11effb5 100644 --- a/src/appdistribution/types.ts +++ b/src/appdistribution/types.ts @@ -123,7 +123,7 @@ export interface ReleaseTest { export interface AiStep { goal: string; hint?: string; - successCriteria?: string; + finalScreenAssertion?: string; } export interface AiInstructions { diff --git a/src/appdistribution/yaml_helper.spec.ts b/src/appdistribution/yaml_helper.spec.ts index a881a7c8e2f..030341c9769 100644 --- a/src/appdistribution/yaml_helper.spec.ts +++ b/src/appdistribution/yaml_helper.spec.ts @@ -16,7 +16,7 @@ const TEST_CASES: TestCase[] = [ { goal: "test-goal", hint: "test-hint", - successCriteria: "test-success-criteria", + finalScreenAssertion: "test-final-screen-assertion", }, ], }, @@ -28,38 +28,41 @@ const TEST_CASES: TestCase[] = [ }, ]; -const YAML_STRING = `- displayName: test-display-name - id: test-case-id - prerequisiteTestCaseId: prerequisite-test-case-id - steps: - - goal: test-goal - hint: test-hint - successCriteria: test-success-criteria -- displayName: minimal-case - id: minimal-id - steps: - - goal: win +const YAML_STRING = `tests: + - displayName: test-display-name + id: test-case-id + prerequisiteTestCaseId: prerequisite-test-case-id + steps: + - goal: test-goal + hint: test-hint + finalScreenAssertion: test-final-screen-assertion + - displayName: minimal-case + id: minimal-id + steps: + - goal: win `; -const YAML_DATA = [ - { - displayName: "test-display-name", - id: "test-case-id", - prerequisiteTestCaseId: "prerequisite-test-case-id", - steps: [ - { - goal: "test-goal", - hint: "test-hint", - successCriteria: "test-success-criteria", - }, - ], - }, - { - displayName: "minimal-case", - id: "minimal-id", - steps: [{ goal: "win" }], - }, -]; +const YAML_DATA = { + tests: [ + { + displayName: "test-display-name", + id: "test-case-id", + prerequisiteTestCaseId: "prerequisite-test-case-id", + steps: [ + { + goal: "test-goal", + hint: "test-hint", + finalScreenAssertion: "test-final-screen-assertion", + }, + ], + }, + { + displayName: "minimal-case", + id: "minimal-id", + steps: [{ goal: "win" }], + }, + ], +}; describe("YamlHelper", () => { it("converts TestCase[] to YAML string", () => { @@ -76,9 +79,10 @@ describe("YamlHelper", () => { it("converts YAML without ID", () => { const testCases = fromYaml( APP_NAME, - `- displayName: minimal-case - steps: - - goal: win + `tests: + - displayName: minimal-case + steps: + - goal: win `, ); expect(testCases).to.eql([ @@ -93,29 +97,35 @@ describe("YamlHelper", () => { expect(() => fromYaml( APP_NAME, - `- steps: - - goal: test-goal - hint: test-hint - successCriteria: test-success-criteria + `tests: + - steps: + - goal: test-goal + hint: test-hint + finalScreenAssertion: test-final-screen-assertion `, ), ).to.throw(/"displayName" is required/); }); it("throws error if steps is missing", () => { - expect(() => fromYaml(APP_NAME, `- displayName: test-display-name`)).to.throw( - /"steps" is required/, - ); + expect(() => + fromYaml( + APP_NAME, + `tests: + - displayName: test-display-name`, + ), + ).to.throw(/"steps" is required/); }); it("throws error if goal is missing", () => { expect(() => fromYaml( APP_NAME, - `- displayName: test-display-name - steps: - - hint: test-hint - successCriteria: test-success-criteria + `tests: + - displayName: test-display-name + steps: + - hint: test-hint + finalScreenAssertion: test-final-screen-assertion `, ), ).to.throw(/"goal" is required/); @@ -125,10 +135,11 @@ describe("YamlHelper", () => { expect(() => fromYaml( APP_NAME, - `- displayName: test-display-name - extraTestCaseProperty: property - steps: - - goal: test-goal + `tests: + - displayName: test-display-name + extraTestCaseProperty: property + steps: + - goal: test-goal `, ), ).to.throw(/unexpected property "extraTestCaseProperty"/); @@ -138,10 +149,11 @@ describe("YamlHelper", () => { expect(() => fromYaml( APP_NAME, - `- displayName: test-display-name - steps: - - goal: test-goal - extraStepProperty: property + `tests: + - displayName: test-display-name + steps: + - goal: test-goal + extraStepProperty: property `, ), ).to.throw(/unexpected property "extraStepProperty"/); @@ -151,13 +163,22 @@ describe("YamlHelper", () => { expect(() => fromYaml( APP_NAME, - `- -this is not valid YAML`, + `tests: + - + invalid key: value`, ), - ).to.throw(/at line 2/); + ).to.throw(/at line 3/); + }); + + it("throws error if YAML doesn't contain a top-level tests field", () => { + expect(() => fromYaml(APP_NAME, "not a list")).to.throw( + /YAML file must contain a top-level 'tests' field with a list of test cases/, + ); }); - it("throws error if YAML doesn't contain a top-level array", () => { - expect(() => fromYaml(APP_NAME, "not a list")).to.throw(/must contain a list of test cases/); + it("throws error if top-level 'tests' field is not an array", () => { + expect(() => fromYaml(APP_NAME, `tests: "not an array"`)).to.throw( + /The 'tests' field in the YAML file must contain a list of test cases/, + ); }); }); diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts index 466467dafdc..b1ba4259584 100644 --- a/src/appdistribution/yaml_helper.ts +++ b/src/appdistribution/yaml_helper.ts @@ -5,10 +5,10 @@ import { TestCase } from "./types"; declare interface YamlStep { goal?: string; hint?: string; - successCriteria?: string; + finalScreenAssertion?: string; } -const ALLOWED_YAML_STEP_KEYS = new Set(["goal", "hint", "successCriteria"]); +const ALLOWED_YAML_STEP_KEYS = new Set(["goal", "hint", "finalScreenAssertion"]); declare interface YamlTestCase { displayName?: string; @@ -38,13 +38,15 @@ function toYamlTestCases(testCases: TestCase[]): YamlTestCase[] { steps: testCase.aiInstructions.steps.map((step) => ({ goal: step.goal, ...(step.hint && { hint: step.hint }), - ...(step.successCriteria && { successCriteria: step.successCriteria }), + ...(step.finalScreenAssertion && { + finalScreenAssertion: step.finalScreenAssertion, + }), })), })); } export function toYaml(testCases: TestCase[]): string { - return jsYaml.safeDump(toYamlTestCases(testCases)); + return jsYaml.safeDump({ tests: toYamlTestCases(testCases) }); } function castExists(it: T | null | undefined, thing: string): T { @@ -73,8 +75,8 @@ function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): Test return { goal: castExists(yamlStep.goal, "goal"), ...(yamlStep.hint && { hint: yamlStep.hint }), - ...(yamlStep.successCriteria && { - successCriteria: yamlStep.successCriteria, + ...(yamlStep.finalScreenAssertion && { + finalScreenAssertion: yamlStep.finalScreenAssertion, }), }; }), @@ -96,8 +98,16 @@ export function fromYaml(appName: string, yaml: string): TestCase[] { } catch (err: unknown) { throw new FirebaseError(`Failed to parse YAML: ${getErrMsg(err)}`); } - if (!Array.isArray(parsedYaml)) { - throw new FirebaseError("YAML file must contain a list of test cases."); + if (!parsedYaml || typeof parsedYaml !== "object" || !("tests" in parsedYaml)) { + throw new FirebaseError( + "YAML file must contain a top-level 'tests' field with a list of test cases.", + ); + } + const yamlTestCases = (parsedYaml as any).tests; + if (!Array.isArray(yamlTestCases)) { + throw new FirebaseError( + "The 'tests' field in the YAML file must contain a list of test cases.", + ); } - return fromYamlTestCases(appName, parsedYaml as YamlTestCase[]); + return fromYamlTestCases(appName, yamlTestCases as YamlTestCase[]); } From b99f98c2f7c5f2416ea1f60a66ef8e945301fb20 Mon Sep 17 00:00:00 2001 From: Lee Kellogg Date: Mon, 9 Feb 2026 23:19:56 -0500 Subject: [PATCH 2/5] Update apptesting MCP yaml format to match test-cases-in-code format (#9900) --- src/mcp/prompts/apptesting/run_test.ts | 2 +- src/mcp/tools/apptesting/tests.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mcp/prompts/apptesting/run_test.ts b/src/mcp/prompts/apptesting/run_test.ts index 767da9e2cb4..7df65edb3e9 100644 --- a/src/mcp/prompts/apptesting/run_test.ts +++ b/src/mcp/prompts/apptesting/run_test.ts @@ -66,7 +66,7 @@ Here are a list of prerequisite steps that must be completed before running a te * Goal (required): In one sentence or less, describe what you want the agent to do in this step. * Hint (optional): Provide additional information to help Gemini understand and navigate your app. - * Success Criteria (optional): Your success criteria should be phrased as an observation, such as 'The screen shows a + * Final Screen Assertion (optional): Your final screen assertion should be phrased as an observation, such as 'The screen shows a success message' or 'The checkout page is visible'. The developer has optionally specified the following description for their test: diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index 1e72e395c36..a2cc8426aac 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -26,7 +26,7 @@ const AiStepSchema = z .string() .optional() .describe("Hint text containing suggestions to help the agent accomplish the goal."), - successCriteria: z + finalScreenAssertion: z .string() .optional() .describe( From 0baf35d7251aa2b13d38f2e200a7bb1d1f788dd6 Mon Sep 17 00:00:00 2001 From: Lee Kellogg Date: Mon, 2 Mar 2026 19:06:10 -0500 Subject: [PATCH 3/5] Keep AiStep type using successCriteria field to match API --- src/appdistribution/types.ts | 2 +- src/appdistribution/yaml_helper.spec.ts | 2 +- src/appdistribution/yaml_helper.ts | 6 +++--- src/apptesting/invokeTests.spec.ts | 4 ++-- src/apptesting/parseTestFiles.spec.ts | 2 +- src/apptesting/types.ts | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/appdistribution/types.ts b/src/appdistribution/types.ts index dcfd11effb5..e0e5e087832 100644 --- a/src/appdistribution/types.ts +++ b/src/appdistribution/types.ts @@ -123,7 +123,7 @@ export interface ReleaseTest { export interface AiStep { goal: string; hint?: string; - finalScreenAssertion?: string; + successCriteria?: string; } export interface AiInstructions { diff --git a/src/appdistribution/yaml_helper.spec.ts b/src/appdistribution/yaml_helper.spec.ts index 030341c9769..345caed66ef 100644 --- a/src/appdistribution/yaml_helper.spec.ts +++ b/src/appdistribution/yaml_helper.spec.ts @@ -16,7 +16,7 @@ const TEST_CASES: TestCase[] = [ { goal: "test-goal", hint: "test-hint", - finalScreenAssertion: "test-final-screen-assertion", + successCriteria: "test-final-screen-assertion", }, ], }, diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts index b1ba4259584..caedb886717 100644 --- a/src/appdistribution/yaml_helper.ts +++ b/src/appdistribution/yaml_helper.ts @@ -38,8 +38,8 @@ function toYamlTestCases(testCases: TestCase[]): YamlTestCase[] { steps: testCase.aiInstructions.steps.map((step) => ({ goal: step.goal, ...(step.hint && { hint: step.hint }), - ...(step.finalScreenAssertion && { - finalScreenAssertion: step.finalScreenAssertion, + ...(step.successCriteria && { + finalScreenAssertion: step.successCriteria, }), })), })); @@ -76,7 +76,7 @@ function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): Test goal: castExists(yamlStep.goal, "goal"), ...(yamlStep.hint && { hint: yamlStep.hint }), ...(yamlStep.finalScreenAssertion && { - finalScreenAssertion: yamlStep.finalScreenAssertion, + successCriteria: yamlStep.finalScreenAssertion, }), }; }), diff --git a/src/apptesting/invokeTests.spec.ts b/src/apptesting/invokeTests.spec.ts index 6ba1e3dbddb..55027bc3e8e 100644 --- a/src/apptesting/invokeTests.spec.ts +++ b/src/apptesting/invokeTests.spec.ts @@ -55,7 +55,7 @@ describe("invokeTests", () => { testCase: { startUri: "https://www.example.com", displayName: "testName2", - steps: [{ goal: "retest it", finalScreenAssertion: "a dialog appears" }], + steps: [{ goal: "retest it", successCriteria: "a dialog appears" }], }, testExecution: [{ config: { browser: Browser.CHROME } }], }, @@ -89,7 +89,7 @@ describe("invokeTests", () => { steps: [ { goal: "retest it", - finalScreenAssertion: "a dialog appears", + successCriteria: "a dialog appears", }, ], startUri: "https://www.example.com", diff --git a/src/apptesting/parseTestFiles.spec.ts b/src/apptesting/parseTestFiles.spec.ts index 494c4aa96c4..f9022c24e3d 100644 --- a/src/apptesting/parseTestFiles.spec.ts +++ b/src/apptesting/parseTestFiles.spec.ts @@ -82,7 +82,7 @@ describe("parseTestFiles", () => { { goal: "View the provided application", hint: "No additional actions should be necessary", - finalScreenAssertion: "The application should load with no obvious errors", + successCriteria: "The application should load with no obvious errors", }, ], }, diff --git a/src/apptesting/types.ts b/src/apptesting/types.ts index a6c6f2d7139..e0fbe8d618b 100644 --- a/src/apptesting/types.ts +++ b/src/apptesting/types.ts @@ -1,6 +1,6 @@ export interface TestStep { goal: string; - finalScreenAssertion?: string; + successCriteria?: string; hint?: string; } From 94bf4cb449d8cc5c9d81bf8a00debf0e354f5702 Mon Sep 17 00:00:00 2001 From: Lee Kellogg Date: Mon, 2 Mar 2026 19:09:55 -0500 Subject: [PATCH 4/5] Add note to MCP prompt that the final screen assertion is required for the last step --- src/mcp/prompts/apptesting/run_test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mcp/prompts/apptesting/run_test.ts b/src/mcp/prompts/apptesting/run_test.ts index 7df65edb3e9..b867ab1cb7a 100644 --- a/src/mcp/prompts/apptesting/run_test.ts +++ b/src/mcp/prompts/apptesting/run_test.ts @@ -66,8 +66,8 @@ Here are a list of prerequisite steps that must be completed before running a te * Goal (required): In one sentence or less, describe what you want the agent to do in this step. * Hint (optional): Provide additional information to help Gemini understand and navigate your app. - * Final Screen Assertion (optional): Your final screen assertion should be phrased as an observation, such as 'The screen shows a - success message' or 'The checkout page is visible'. + * Final Screen Assertion (required for last step): Your final screen assertion should be phrased as an observation, such as 'The screen shows a + success message' or 'The checkout page is visible'. Optional except for the last step, for which it is required. The developer has optionally specified the following description for their test: * ${testDescription} From a19cea67d3fc664f5e1220d7a3d51326784706cb Mon Sep 17 00:00:00 2001 From: Lee Kellogg Date: Mon, 2 Mar 2026 19:14:18 -0500 Subject: [PATCH 5/5] Revert WATA changes --- src/apptesting/invokeTests.spec.ts | 4 ++-- src/apptesting/parseTestFiles.spec.ts | 2 +- src/apptesting/types.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/apptesting/invokeTests.spec.ts b/src/apptesting/invokeTests.spec.ts index 55027bc3e8e..6ba1e3dbddb 100644 --- a/src/apptesting/invokeTests.spec.ts +++ b/src/apptesting/invokeTests.spec.ts @@ -55,7 +55,7 @@ describe("invokeTests", () => { testCase: { startUri: "https://www.example.com", displayName: "testName2", - steps: [{ goal: "retest it", successCriteria: "a dialog appears" }], + steps: [{ goal: "retest it", finalScreenAssertion: "a dialog appears" }], }, testExecution: [{ config: { browser: Browser.CHROME } }], }, @@ -89,7 +89,7 @@ describe("invokeTests", () => { steps: [ { goal: "retest it", - successCriteria: "a dialog appears", + finalScreenAssertion: "a dialog appears", }, ], startUri: "https://www.example.com", diff --git a/src/apptesting/parseTestFiles.spec.ts b/src/apptesting/parseTestFiles.spec.ts index f9022c24e3d..494c4aa96c4 100644 --- a/src/apptesting/parseTestFiles.spec.ts +++ b/src/apptesting/parseTestFiles.spec.ts @@ -82,7 +82,7 @@ describe("parseTestFiles", () => { { goal: "View the provided application", hint: "No additional actions should be necessary", - successCriteria: "The application should load with no obvious errors", + finalScreenAssertion: "The application should load with no obvious errors", }, ], }, diff --git a/src/apptesting/types.ts b/src/apptesting/types.ts index e0fbe8d618b..a6c6f2d7139 100644 --- a/src/apptesting/types.ts +++ b/src/apptesting/types.ts @@ -1,6 +1,6 @@ export interface TestStep { goal: string; - successCriteria?: string; + finalScreenAssertion?: string; hint?: string; }