Update yaml format for appdistribution:testcases:import/export and MCP

src/appdistribution/yaml_helper.spec.ts

@@ -16,7 +16,7 @@ const TEST_CASES: TestCase[] = [
         {
           goal: "test-goal",
           hint: "test-hint",
-          successCriteria: "test-success-criteria",
+          successCriteria: "test-final-screen-assertion",
         },
       ],
     },
@@ -28,38 +28,41 @@ const TEST_CASES: TestCase[] = [
   },
 ];
 
-const YAML_STRING = `- displayName: test-display-name
-  id: test-case-id
-  prerequisiteTestCaseId: prerequisite-test-case-id
-  steps:
-    - goal: test-goal
-      hint: test-hint
-      successCriteria: test-success-criteria
-- displayName: minimal-case
-  id: minimal-id
-  steps:
-    - goal: win
+const YAML_STRING = `tests:
+  - displayName: test-display-name
+    id: test-case-id
+    prerequisiteTestCaseId: prerequisite-test-case-id
+    steps:
+      - goal: test-goal
+        hint: test-hint
+        finalScreenAssertion: test-final-screen-assertion
+  - displayName: minimal-case
+    id: minimal-id
+    steps:
+      - goal: win
 `;
 
-const YAML_DATA = [
-  {
-    displayName: "test-display-name",
-    id: "test-case-id",
-    prerequisiteTestCaseId: "prerequisite-test-case-id",
-    steps: [
-      {
-        goal: "test-goal",
-        hint: "test-hint",
-        successCriteria: "test-success-criteria",
-      },
-    ],
-  },
-  {
-    displayName: "minimal-case",
-    id: "minimal-id",
-    steps: [{ goal: "win" }],
-  },
-];
+const YAML_DATA = {
+  tests: [
+    {
+      displayName: "test-display-name",
+      id: "test-case-id",
+      prerequisiteTestCaseId: "prerequisite-test-case-id",
+      steps: [
+        {
+          goal: "test-goal",
+          hint: "test-hint",
+          finalScreenAssertion: "test-final-screen-assertion",
+        },
+      ],
+    },
+    {
+      displayName: "minimal-case",
+      id: "minimal-id",
+      steps: [{ goal: "win" }],
+    },
+  ],
+};
 
 describe("YamlHelper", () => {
   it("converts TestCase[] to YAML string", () => {
@@ -76,9 +79,10 @@ describe("YamlHelper", () => {
   it("converts YAML without ID", () => {
     const testCases = fromYaml(
       APP_NAME,
-      `- displayName: minimal-case
-  steps:
-    - goal: win
+      `tests:
+  - displayName: minimal-case
+    steps:
+      - goal: win
 `,
     );
     expect(testCases).to.eql([
@@ -93,29 +97,35 @@ describe("YamlHelper", () => {
     expect(() =>
       fromYaml(
         APP_NAME,
-        `- steps:
-  - goal: test-goal
-    hint: test-hint
-    successCriteria: test-success-criteria
+        `tests:
+  - steps:
+      - goal: test-goal
+        hint: test-hint
+        finalScreenAssertion: test-final-screen-assertion
 `,
       ),
     ).to.throw(/"displayName" is required/);
   });
 
   it("throws error if steps is missing", () => {
-    expect(() => fromYaml(APP_NAME, `- displayName: test-display-name`)).to.throw(
-      /"steps" is required/,
-    );
+    expect(() =>
+      fromYaml(
+        APP_NAME,
+        `tests:
+  - displayName: test-display-name`,
+      ),
+    ).to.throw(/"steps" is required/);
   });
 
   it("throws error if goal is missing", () => {
     expect(() =>
       fromYaml(
         APP_NAME,
-        `- displayName: test-display-name
-  steps:
-    - hint: test-hint
-      successCriteria: test-success-criteria
+        `tests:
+  - displayName: test-display-name
+    steps:
+      - hint: test-hint
+        finalScreenAssertion: test-final-screen-assertion
 `,
       ),
     ).to.throw(/"goal" is required/);
@@ -125,10 +135,11 @@ describe("YamlHelper", () => {
     expect(() =>
       fromYaml(
         APP_NAME,
-        `- displayName: test-display-name
-  extraTestCaseProperty: property
-  steps:
-    - goal: test-goal
+        `tests:
+  - displayName: test-display-name
+    extraTestCaseProperty: property
+    steps:
+      - goal: test-goal
 `,
       ),
     ).to.throw(/unexpected property "extraTestCaseProperty"/);
@@ -138,10 +149,11 @@ describe("YamlHelper", () => {
     expect(() =>
       fromYaml(
         APP_NAME,
-        `- displayName: test-display-name
-  steps:
-    - goal: test-goal
-      extraStepProperty: property
+        `tests:
+  - displayName: test-display-name
+    steps:
+      - goal: test-goal
+        extraStepProperty: property
 `,
       ),
     ).to.throw(/unexpected property "extraStepProperty"/);
@@ -151,13 +163,22 @@ describe("YamlHelper", () => {
     expect(() =>
       fromYaml(
         APP_NAME,
-        `-
-this is not valid YAML`,
+        `tests:
+  -
+  invalid key: value`,
       ),
-    ).to.throw(/at line 2/);
+    ).to.throw(/at line 3/);
+  });
+
+  it("throws error if YAML doesn't contain a top-level tests field", () => {
+    expect(() => fromYaml(APP_NAME, "not a list")).to.throw(
+      /YAML file must contain a top-level 'tests' field with a list of test cases/,
+    );
   });
 
-  it("throws error if YAML doesn't contain a top-level array", () => {
-    expect(() => fromYaml(APP_NAME, "not a list")).to.throw(/must contain a list of test cases/);
+  it("throws error if top-level 'tests' field is not an array", () => {
+    expect(() => fromYaml(APP_NAME, `tests: "not an array"`)).to.throw(
+      /The 'tests' field in the YAML file must contain a list of test cases/,
+    );
   });
 });

src/appdistribution/yaml_helper.ts

@@ -5,10 +5,10 @@
 declare interface YamlStep {
   goal?: string;
   hint?: string;
-  successCriteria?: string;
+  finalScreenAssertion?: string;
 }
 
-const ALLOWED_YAML_STEP_KEYS = new Set(["goal", "hint", "successCriteria"]);
+const ALLOWED_YAML_STEP_KEYS = new Set(["goal", "hint", "finalScreenAssertion"]);
 
 declare interface YamlTestCase {
   displayName?: string;
@@ -31,30 +31,32 @@
 function toYamlTestCases(testCases: TestCase[]): YamlTestCase[] {
   return testCases.map((testCase) => ({
     displayName: testCase.displayName,
     id: extractIdFromResourceName(testCase.name!), // resource name is retured by server
     ...(testCase.prerequisiteTestCase && {
       prerequisiteTestCaseId: extractIdFromResourceName(testCase.prerequisiteTestCase),
     }),
     steps: testCase.aiInstructions.steps.map((step) => ({
       goal: step.goal,
       ...(step.hint && { hint: step.hint }),
-      ...(step.successCriteria && { successCriteria: step.successCriteria }),
+      ...(step.successCriteria && {
+        finalScreenAssertion: step.successCriteria,
+      }),
     })),
   }));
 }
 
 export function toYaml(testCases: TestCase[]): string {
-  return jsYaml.safeDump(toYamlTestCases(testCases));
+  return jsYaml.safeDump({ tests: toYamlTestCases(testCases) });
 }
 
 function castExists<T>(it: T | null | undefined, thing: string): T {
   if (it == null) {
     throw new FirebaseError(`"${thing}" is required`);
   }
   return it!;
 }
 
 function checkAllowedKeys(allowedKeys: Set<string>, o: object) {
   for (const key of Object.keys(o)) {
     if (!allowedKeys.has(key)) {
       throw new FirebaseError(`unexpected property "${key}"`);
@@ -73,8 +75,8 @@
           return {
             goal: castExists(yamlStep.goal, "goal"),
             ...(yamlStep.hint && { hint: yamlStep.hint }),
-            ...(yamlStep.successCriteria && {
-              successCriteria: yamlStep.successCriteria,
+            ...(yamlStep.finalScreenAssertion && {
+              successCriteria: yamlStep.finalScreenAssertion,
             }),
           };
         }),
@@ -89,15 +91,23 @@
   });
 }
 
 export function fromYaml(appName: string, yaml: string): TestCase[] {
   let parsedYaml: unknown;
   try {
     parsedYaml = jsYaml.safeLoad(yaml);
   } catch (err: unknown) {
     throw new FirebaseError(`Failed to parse YAML: ${getErrMsg(err)}`);
   }
-  if (!Array.isArray(parsedYaml)) {
-    throw new FirebaseError("YAML file must contain a list of test cases.");
+  if (!parsedYaml || typeof parsedYaml !== "object" || !("tests" in parsedYaml)) {
+    throw new FirebaseError(
+      "YAML file must contain a top-level 'tests' field with a list of test cases.",
+    );
+  }
+  const yamlTestCases = (parsedYaml as any).tests;
+  if (!Array.isArray(yamlTestCases)) {
+    throw new FirebaseError(
+      "The 'tests' field in the YAML file must contain a list of test cases.",
+    );
   }
-  return fromYamlTestCases(appName, parsedYaml as YamlTestCase[]);
+  return fromYamlTestCases(appName, yamlTestCases as YamlTestCase[]);
 }

src/mcp/prompts/apptesting/run_test.ts

@@ -66,8 +66,8 @@
 
 	* Goal (required): In one sentence or less, describe what you want the agent to do in this step.
 	* Hint (optional): Provide additional information to help Gemini understand and navigate your app.
-	* Success Criteria (optional): Your success criteria should be phrased as an observation, such as 'The screen shows a
-    success message' or 'The checkout page is visible'.
+	* Final Screen Assertion (required for last step): Your final screen assertion should be phrased as an observation, such as 'The screen shows a
+    success message' or 'The checkout page is visible'. Optional except for the last step, for which it is required.
 
   The developer has optionally specified the following description for their test:
   * ${testDescription}
@@ -113,7 +113,7 @@
       devices.
 
   Once the test has started, provide the developer a link to see the results of the test in the Firebase Console.
   You should already know the value of \`appId\' and \`projectId\` from earlier (if you only know \`projectNumber\',
   use the \`firebase_get_project\` tool to get \`projectId\`). \`packageName\` is the package name of the app we tested. 
   The \`apptesting_run_test\` tool returns a response with field \`name\` in the form 
   projects/{projectNumber}/apps/{appId}/releases/{releaseId}/tests/{releaseTestId}. Extract the values for \'releaseId\' 

src/mcp/tools/apptesting/tests.ts

@@ -26,7 +26,7 @@ const AiStepSchema = z
       .string()
       .optional()
       .describe("Hint text containing suggestions to help the agent accomplish the goal."),
-    successCriteria: z
+    finalScreenAssertion: z
       .string()
       .optional()
       .describe(