diff --git a/src/appdistribution/yaml_helper.spec.ts b/src/appdistribution/yaml_helper.spec.ts index a881a7c8e2f..345caed66ef 100644 --- a/src/appdistribution/yaml_helper.spec.ts +++ b/src/appdistribution/yaml_helper.spec.ts @@ -16,7 +16,7 @@ const TEST_CASES: TestCase[] = [ { goal: "test-goal", hint: "test-hint", - successCriteria: "test-success-criteria", + successCriteria: "test-final-screen-assertion", }, ], }, @@ -28,38 +28,41 @@ const TEST_CASES: TestCase[] = [ }, ]; -const YAML_STRING = `- displayName: test-display-name - id: test-case-id - prerequisiteTestCaseId: prerequisite-test-case-id - steps: - - goal: test-goal - hint: test-hint - successCriteria: test-success-criteria -- displayName: minimal-case - id: minimal-id - steps: - - goal: win +const YAML_STRING = `tests: + - displayName: test-display-name + id: test-case-id + prerequisiteTestCaseId: prerequisite-test-case-id + steps: + - goal: test-goal + hint: test-hint + finalScreenAssertion: test-final-screen-assertion + - displayName: minimal-case + id: minimal-id + steps: + - goal: win `; -const YAML_DATA = [ - { - displayName: "test-display-name", - id: "test-case-id", - prerequisiteTestCaseId: "prerequisite-test-case-id", - steps: [ - { - goal: "test-goal", - hint: "test-hint", - successCriteria: "test-success-criteria", - }, - ], - }, - { - displayName: "minimal-case", - id: "minimal-id", - steps: [{ goal: "win" }], - }, -]; +const YAML_DATA = { + tests: [ + { + displayName: "test-display-name", + id: "test-case-id", + prerequisiteTestCaseId: "prerequisite-test-case-id", + steps: [ + { + goal: "test-goal", + hint: "test-hint", + finalScreenAssertion: "test-final-screen-assertion", + }, + ], + }, + { + displayName: "minimal-case", + id: "minimal-id", + steps: [{ goal: "win" }], + }, + ], +}; describe("YamlHelper", () => { it("converts TestCase[] to YAML string", () => { @@ -76,9 +79,10 @@ describe("YamlHelper", () => { it("converts YAML without ID", () => { const testCases = fromYaml( APP_NAME, - `- displayName: minimal-case - steps: - - goal: win + `tests: + - displayName: minimal-case + steps: + - goal: win `, ); expect(testCases).to.eql([ @@ -93,29 +97,35 @@ describe("YamlHelper", () => { expect(() => fromYaml( APP_NAME, - `- steps: - - goal: test-goal - hint: test-hint - successCriteria: test-success-criteria + `tests: + - steps: + - goal: test-goal + hint: test-hint + finalScreenAssertion: test-final-screen-assertion `, ), ).to.throw(/"displayName" is required/); }); it("throws error if steps is missing", () => { - expect(() => fromYaml(APP_NAME, `- displayName: test-display-name`)).to.throw( - /"steps" is required/, - ); + expect(() => + fromYaml( + APP_NAME, + `tests: + - displayName: test-display-name`, + ), + ).to.throw(/"steps" is required/); }); it("throws error if goal is missing", () => { expect(() => fromYaml( APP_NAME, - `- displayName: test-display-name - steps: - - hint: test-hint - successCriteria: test-success-criteria + `tests: + - displayName: test-display-name + steps: + - hint: test-hint + finalScreenAssertion: test-final-screen-assertion `, ), ).to.throw(/"goal" is required/); @@ -125,10 +135,11 @@ describe("YamlHelper", () => { expect(() => fromYaml( APP_NAME, - `- displayName: test-display-name - extraTestCaseProperty: property - steps: - - goal: test-goal + `tests: + - displayName: test-display-name + extraTestCaseProperty: property + steps: + - goal: test-goal `, ), ).to.throw(/unexpected property "extraTestCaseProperty"/); @@ -138,10 +149,11 @@ describe("YamlHelper", () => { expect(() => fromYaml( APP_NAME, - `- displayName: test-display-name - steps: - - goal: test-goal - extraStepProperty: property + `tests: + - displayName: test-display-name + steps: + - goal: test-goal + extraStepProperty: property `, ), ).to.throw(/unexpected property "extraStepProperty"/); @@ -151,13 +163,22 @@ describe("YamlHelper", () => { expect(() => fromYaml( APP_NAME, - `- -this is not valid YAML`, + `tests: + - + invalid key: value`, ), - ).to.throw(/at line 2/); + ).to.throw(/at line 3/); + }); + + it("throws error if YAML doesn't contain a top-level tests field", () => { + expect(() => fromYaml(APP_NAME, "not a list")).to.throw( + /YAML file must contain a top-level 'tests' field with a list of test cases/, + ); }); - it("throws error if YAML doesn't contain a top-level array", () => { - expect(() => fromYaml(APP_NAME, "not a list")).to.throw(/must contain a list of test cases/); + it("throws error if top-level 'tests' field is not an array", () => { + expect(() => fromYaml(APP_NAME, `tests: "not an array"`)).to.throw( + /The 'tests' field in the YAML file must contain a list of test cases/, + ); }); }); diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts index 466467dafdc..caedb886717 100644 --- a/src/appdistribution/yaml_helper.ts +++ b/src/appdistribution/yaml_helper.ts @@ -5,10 +5,10 @@ import { TestCase } from "./types"; declare interface YamlStep { goal?: string; hint?: string; - successCriteria?: string; + finalScreenAssertion?: string; } -const ALLOWED_YAML_STEP_KEYS = new Set(["goal", "hint", "successCriteria"]); +const ALLOWED_YAML_STEP_KEYS = new Set(["goal", "hint", "finalScreenAssertion"]); declare interface YamlTestCase { displayName?: string; @@ -38,13 +38,15 @@ function toYamlTestCases(testCases: TestCase[]): YamlTestCase[] { steps: testCase.aiInstructions.steps.map((step) => ({ goal: step.goal, ...(step.hint && { hint: step.hint }), - ...(step.successCriteria && { successCriteria: step.successCriteria }), + ...(step.successCriteria && { + finalScreenAssertion: step.successCriteria, + }), })), })); } export function toYaml(testCases: TestCase[]): string { - return jsYaml.safeDump(toYamlTestCases(testCases)); + return jsYaml.safeDump({ tests: toYamlTestCases(testCases) }); } function castExists(it: T | null | undefined, thing: string): T { @@ -73,8 +75,8 @@ function fromYamlTestCases(appName: string, yamlTestCases: YamlTestCase[]): Test return { goal: castExists(yamlStep.goal, "goal"), ...(yamlStep.hint && { hint: yamlStep.hint }), - ...(yamlStep.successCriteria && { - successCriteria: yamlStep.successCriteria, + ...(yamlStep.finalScreenAssertion && { + successCriteria: yamlStep.finalScreenAssertion, }), }; }), @@ -96,8 +98,16 @@ export function fromYaml(appName: string, yaml: string): TestCase[] { } catch (err: unknown) { throw new FirebaseError(`Failed to parse YAML: ${getErrMsg(err)}`); } - if (!Array.isArray(parsedYaml)) { - throw new FirebaseError("YAML file must contain a list of test cases."); + if (!parsedYaml || typeof parsedYaml !== "object" || !("tests" in parsedYaml)) { + throw new FirebaseError( + "YAML file must contain a top-level 'tests' field with a list of test cases.", + ); + } + const yamlTestCases = (parsedYaml as any).tests; + if (!Array.isArray(yamlTestCases)) { + throw new FirebaseError( + "The 'tests' field in the YAML file must contain a list of test cases.", + ); } - return fromYamlTestCases(appName, parsedYaml as YamlTestCase[]); + return fromYamlTestCases(appName, yamlTestCases as YamlTestCase[]); } diff --git a/src/mcp/prompts/apptesting/run_test.ts b/src/mcp/prompts/apptesting/run_test.ts index 767da9e2cb4..b867ab1cb7a 100644 --- a/src/mcp/prompts/apptesting/run_test.ts +++ b/src/mcp/prompts/apptesting/run_test.ts @@ -66,8 +66,8 @@ Here are a list of prerequisite steps that must be completed before running a te * Goal (required): In one sentence or less, describe what you want the agent to do in this step. * Hint (optional): Provide additional information to help Gemini understand and navigate your app. - * Success Criteria (optional): Your success criteria should be phrased as an observation, such as 'The screen shows a - success message' or 'The checkout page is visible'. + * Final Screen Assertion (required for last step): Your final screen assertion should be phrased as an observation, such as 'The screen shows a + success message' or 'The checkout page is visible'. Optional except for the last step, for which it is required. The developer has optionally specified the following description for their test: * ${testDescription} diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index 1e72e395c36..a2cc8426aac 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -26,7 +26,7 @@ const AiStepSchema = z .string() .optional() .describe("Hint text containing suggestions to help the agent accomplish the goal."), - successCriteria: z + finalScreenAssertion: z .string() .optional() .describe(