Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Tier 0 (offline) — verify `azd ai agent code download --help` prints usage.
name: "code-download-help"
command: "azd ai agent code download --help"
cwd: "/tmp"
tags: ["tier:0", "cmd:code", "parallel-safe"]

goals:
- "Wait for the help text to render."
- "Confirm the output includes the command description, usage line, examples, and flags (--dest, --version, --zip)."
- "Confirm the process exits cleanly (exit 0) without error or stack trace."
- "Take a screenshot of the help output."
- "Report a finding if help text is missing, incomplete, or the command errors."
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Tier 0 (offline) — verify `azd ai agent delete --help` prints usage.
name: "delete-help"
command: "azd ai agent delete --help"
cwd: "/tmp"
tags: ["tier:0", "cmd:delete", "parallel-safe"]

goals:
- "Wait for the help text to render."
- "Confirm the output includes the command description, usage line, examples, and flags (--force, --version)."
- "Confirm the process exits cleanly (exit 0) without error or stack trace."
- "Take a screenshot of the help output."
- "Report a finding if help text is missing, incomplete, or the command errors."
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Tier 0 (offline) — verify `azd ai agent endpoint show --help` prints usage.
name: "endpoint-show-help"
command: "azd ai agent endpoint show --help"
cwd: "/tmp"
tags: ["tier:0", "cmd:endpoint", "parallel-safe"]

goals:
- "Wait for the help text to render."
- "Confirm the output includes the command description, usage line, examples, and supported output formats (json, table)."
- "Confirm the process exits cleanly (exit 0) without error or stack trace."
- "Take a screenshot of the help output."
- "Report a finding if help text is missing, incomplete, or the command errors."
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Tier 2 (cloud E2E) — `endpoint show` displays the deployed agent's endpoint config.
#
# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully.
name: "endpoint-show"
command: "azd ai agent endpoint show"
cwd: "~/working/azd-agents-shared/{shared_agent_name}"
tags: ["tier:2", "cmd:endpoint", "serial-only"]

pre:
- run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'"
cwd: "~/working/azd-agents-shared/{shared_agent_name}"
name: "assert shared agent is deployed"
continue_on_error: true

goals:
- "Wait for the endpoint configuration table to render (auto-resolves agent from azure.yaml)."
- "Confirm the output shows protocol(s) configured on the agent (e.g. 'responses')."
- "Confirm the endpoint URL or base path is displayed."
- "Run the command again with '--output json' and confirm valid JSON is returned with protocol and endpoint fields."
- "Take a screenshot of both the table and JSON outputs."
- "Report a finding if the command errors, shows no endpoint config, or returns unexpected empty fields for a deployed agent."
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Tier 2 (cloud E2E) — `code download` against a container-based agent (negative path).
#
# The shared agent uses Docker (container) deploy mode, not code_configuration.
# `code download` is only supported for code-based agents. This scenario verifies
# the CLI returns a clear, actionable error (AgentNotCodeBased / 409) rather than
# a confusing or generic failure.
#
# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully.
name: "code-download-not-code-based"
command: "azd ai agent code download --dest ~/working/azd-agents-shared/code-download-test"
cwd: "~/working/azd-agents-shared/{shared_agent_name}"
tags: ["tier:2", "cmd:code", "serial-only", "negative-path"]

pre:
- run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'"
cwd: "~/working/azd-agents-shared/{shared_agent_name}"
name: "assert shared agent is deployed"
continue_on_error: true

goals:
- "Run code download against the shared agent (which is container-based, NOT code-based)."
- "Confirm the command returns a non-zero exit code and a clear error message indicating the agent is not code-based (e.g. 'AgentNotCodeBased' or 'not a code-based agent')."
- "Confirm the error message is user-friendly and actionable — not a raw stack trace or generic HTTP error."
- "Take a screenshot of the error output."
- "Report a finding if the command succeeds unexpectedly, hangs, panics, or gives a confusing/generic error message instead of a clear 'not code-based' explanation."
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Tier 2 (cloud E2E) — `delete` removes the shared agent from Foundry.
#
# Precondition: 20-setup-deploy-shared-agent.yaml has been run successfully.
# NOTE: This MUST run AFTER all other Tier 2 scenarios that use the shared agent
# and BEFORE 2Z-teardown-down.yaml. Ordering: 2A → 2B → 2C → 2D → 2Z.
name: "delete"
command: "azd ai agent delete --force"
cwd: "~/working/azd-agents-shared/{shared_agent_name}"
tags: ["tier:2", "cmd:delete", "serial-only"]

pre:
- run: "test -f ~/working/azd-agents-shared/{shared_agent_name}/azure.yaml && echo 'OK: shared agent project found' || echo 'PRECONDITION FAILED: shared agent not found - run 20-setup-deploy-shared-agent.yaml first'"
cwd: "~/working/azd-agents-shared/{shared_agent_name}"
name: "assert shared agent is deployed"
continue_on_error: true

goals:
- "Wait for the delete command to complete (auto-resolves agent from azure.yaml, --force skips confirmation and handles active sessions)."
- "Confirm the command reports the agent was deleted successfully (exit 0)."
- "After deletion, run 'azd ai agent show' in the same directory and confirm it returns an error or indicates the agent no longer exists."
- "Take a screenshot of the delete result and the post-delete show output."
- "Report a finding if delete fails, hangs, or if 'show' still reports the agent as active after deletion."