From c03a30866c5904077e0fb06bf8d280e3fbf52a4b Mon Sep 17 00:00:00 2001 From: Jian Wu Date: Wed, 10 Jun 2026 17:04:01 +0800 Subject: [PATCH] Add test scenarios for delete, endpoint show, and code download Add 6 new cli-interactive-tester scenarios covering three recently merged commands: Tier 0 (offline help validation): - 00-delete-help.yaml: validates azd ai agent delete --help output - 00-endpoint-show-help.yaml: validates azd ai agent endpoint show --help output - 00-code-download-help.yaml: validates azd ai agent code download --help output Tier 2 (cloud E2E, run between 2A-doctor and 2Z-teardown): - 2B-endpoint-show.yaml: shows endpoint config (table + JSON output) - 2C-code-download.yaml: negative-path test (container agent returns AgentNotCodeBased) - 2D-delete.yaml: deletes agent with --force, confirms removal via show All scenarios tested locally: 6/6 PASS. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../00-code-download-help.yaml | 12 +++++++++ .../00-delete-help.yaml | 12 +++++++++ .../00-endpoint-show-help.yaml | 12 +++++++++ .../2B-endpoint-show.yaml | 21 ++++++++++++++++ .../2C-code-download.yaml | 25 +++++++++++++++++++ .../2D-delete.yaml | 22 ++++++++++++++++ 6 files changed, 104 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-code-download-help.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-delete-help.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-endpoint-show-help.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml create mode 100644 cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2D-delete.yaml diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-code-download-help.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-code-download-help.yaml new file mode 100644 index 00000000000..7d47cdef583 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-code-download-help.yaml @@ -0,0 +1,12 @@ +# Tier 0 (offline) — verify `azd ai agent code download --help` prints usage. +name: "code-download-help" +command: "azd ai agent code download --help" +cwd: "/tmp" +tags: ["tier:0", "cmd:code", "parallel-safe"] + +goals: + - "Wait for the help text to render." + - "Confirm the output includes the command description, usage line, examples, and flags (--dest, --version, --zip)." + - "Confirm the process exits cleanly (exit 0) without error or stack trace." + - "Take a screenshot of the help output." + - "Report a finding if help text is missing, incomplete, or the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-delete-help.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-delete-help.yaml new file mode 100644 index 00000000000..cbdab56b8ce --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-delete-help.yaml @@ -0,0 +1,12 @@ +# Tier 0 (offline) — verify `azd ai agent delete --help` prints usage. +name: "delete-help" +command: "azd ai agent delete --help" +cwd: "/tmp" +tags: ["tier:0", "cmd:delete", "parallel-safe"] + +goals: + - "Wait for the help text to render." + - "Confirm the output includes the command description, usage line, examples, and flags (--force, --version)." + - "Confirm the process exits cleanly (exit 0) without error or stack trace." + - "Take a screenshot of the help output." + - "Report a finding if help text is missing, incomplete, or the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-endpoint-show-help.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-endpoint-show-help.yaml new file mode 100644 index 00000000000..d0b598108f0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/00-endpoint-show-help.yaml @@ -0,0 +1,12 @@ +# Tier 0 (offline) — verify `azd ai agent endpoint show --help` prints usage. +name: "endpoint-show-help" +command: "azd ai agent endpoint show --help" +cwd: "/tmp" +tags: ["tier:0", "cmd:endpoint", "parallel-safe"] + +goals: + - "Wait for the help text to render." + - "Confirm the output includes the command description, usage line, examples, and supported output formats (json, table)." + - "Confirm the process exits cleanly (exit 0) without error or stack trace." + - "Take a screenshot of the help output." + - "Report a finding if help text is missing, incomplete, or the command errors." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml new file mode 100644 index 00000000000..756108a70a2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2B-endpoint-show.yaml @@ -0,0 +1,21 @@ +# Tier 2 (cloud E2E) — `endpoint show` displays the deployed agent's endpoint config. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "endpoint-show" +command: "azd ai agent endpoint show" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:endpoint", "serial-only"] + +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the endpoint configuration table to render (auto-resolves agent from azure.yaml)." + - "Confirm the output shows protocol(s) configured on the agent (e.g. 'responses')." + - "Confirm the endpoint URL or base path is displayed." + - "Run the command again with '--output json' and confirm valid JSON is returned with protocol and endpoint fields." + - "Take a screenshot of both the table and JSON outputs." + - "Report a finding if the command errors, shows no endpoint config, or returns unexpected empty fields for a deployed agent." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml new file mode 100644 index 00000000000..e2cd3087dd8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2C-code-download.yaml @@ -0,0 +1,25 @@ +# Tier 2 (cloud E2E) — `code download` against a container-based agent (negative path). +# +# The shared agent uses Docker (container) deploy mode, not code_configuration. +# `code download` is only supported for code-based agents. This scenario verifies +# the CLI returns a clear, actionable error (AgentNotCodeBased / 409) rather than +# a confusing or generic failure. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +name: "code-download-not-code-based" +command: "azd ai agent code download --dest ~/working/azd-agents-shared/code-download-test" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:code", "serial-only", "negative-path"] + +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Run code download against the shared agent (which is container-based, NOT code-based)." + - "Confirm the command returns a non-zero exit code and a clear error message indicating the agent is not code-based (e.g. 'AgentNotCodeBased' or 'not a code-based agent')." + - "Confirm the error message is user-friendly and actionable — not a raw stack trace or generic HTTP error." + - "Take a screenshot of the error output." + - "Report a finding if the command succeeds unexpectedly, hangs, panics, or gives a confusing/generic error message instead of a clear 'not code-based' explanation." diff --git a/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2D-delete.yaml b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2D-delete.yaml new file mode 100644 index 00000000000..3bb1deaf2ed --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/tests/cli-interactive-tester-scenarios/2D-delete.yaml @@ -0,0 +1,22 @@ +# Tier 2 (cloud E2E) — `delete` removes the shared agent from Foundry. +# +# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully. +# NOTE: This MUST run AFTER all other Tier 2 scenarios that use the shared agent +# and BEFORE 2Z-teardown-down.yaml. Ordering: 2A → 2B → 2C → 2D → 2Z. +name: "delete" +command: "azd ai agent delete --force" +cwd: "~/working/azd-agents-shared/{shared_agent_name}" +tags: ["tier:2", "cmd:delete", "serial-only"] + +pre: + - run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'" + cwd: "~/working/azd-agents-shared/{shared_agent_name}" + name: "assert shared agent is deployed" + continue_on_error: true + +goals: + - "Wait for the delete command to complete (auto-resolves agent from azure.yaml, --force skips confirmation and handles active sessions)." + - "Confirm the command reports the agent was deleted successfully (exit 0)." + - "After deletion, run 'azd ai agent show' in the same directory and confirm it returns an error or indicates the agent no longer exists." + - "Take a screenshot of the delete result and the post-delete show output." + - "Report a finding if delete fails, hangs, or if 'show' still reports the agent as active after deletion."