From 518165946ac7795dc175ab86a92d008b4d346930 Mon Sep 17 00:00:00 2001 From: Glenn Harper Date: Thu, 30 Jul 2026 12:52:13 -0400 Subject: [PATCH 1/5] test(azure.ai.agents): validate azure.yaml examples in docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `azure.yaml` examples in this extension's docs weren't validated by anything, so they could drift out of sync with the code and ship broken. Two instances were found by hand recently: the README migration example omitted the required agent `name` (#9328), and a Learn article documented `rai_config.rai_policy_name`, which azd ignores entirely — deploying with no guardrail and no error. Adds TestDocExamplesAreValid, which extracts every fenced YAML block declaring an `azure.ai.agent` service from the extension's markdown and applies two checks: 1. Resolver — the snippet must survive AgentDefinitionFromService, the same entry point azd uses at deploy time. Catches the missing-`name` class. 2. Vocabulary — every property must be declared in schemas/azure.ai.agent.json or parsed by azd core. azd deliberately ignores unrecognized service properties for forward compatibility, which is exactly how a doc can advertise a setting that silently does nothing. Catches the `rai_config` class, which the resolver alone cannot. Not every snippet is meant to be complete: the three `azure.ai.agent` entries in docs/private-networking.md intentionally omit `kind` so azd falls back to the on-disk agent.yaml. Those opt out of check 1 with an `` marker, keeping the default strict rather than inferring intent. Fixes #9330 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e7ba2e0e-80cf-4226-b56e-ac7cbbc7338f --- cli/azd/extensions/azure.ai.agents/AGENTS.md | 28 ++ .../docs/private-networking.md | 3 + .../internal/project/doc_examples_test.go | 372 ++++++++++++++++++ 3 files changed, 403 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go diff --git a/cli/azd/extensions/azure.ai.agents/AGENTS.md b/cli/azd/extensions/azure.ai.agents/AGENTS.md index 16a7e065acf..39046d111d4 100644 --- a/cli/azd/extensions/azure.ai.agents/AGENTS.md +++ b/cli/azd/extensions/azure.ai.agents/AGENTS.md @@ -38,6 +38,34 @@ replace github.com/azure/azure-dev/cli/azd => ../../ That `replace` points this extension at your local `cli/azd` checkout instead of the version in `go.mod`. Do not merge the extension with that `replace` still present. +## Documentation examples + +`azure.yaml` examples in this extension's markdown docs are validated by +`TestDocExamplesAreValid` (`internal/project/doc_examples_test.go`). Every fenced +YAML block declaring an `azure.ai.agent` service must: + +1. Resolve through `AgentDefinitionFromService` without error, and +2. Use only properties declared in `schemas/azure.ai.agent.json` or parsed by azd + core. azd ignores unknown service properties at runtime, so an undocumented + key deploys cleanly while doing nothing — the test blocks that in our docs. + +Snippets that are deliberately incomplete (for example, the network examples in +`docs/private-networking.md`, which omit `kind` because azd falls back to the +on-disk `agent.yaml`) opt out of the "must fully resolve" check with a marker on +the line before the fence: + +````markdown + +```yaml +services: + my-agent: + host: azure.ai.agent +``` +```` + +Use the marker only when the snippet is intentionally partial. If a complete +example fails, fix the example rather than adding the marker. + ## Error handling This extension uses `internal/exterrors` so the azd host can show a useful message, attach an optional suggestion, and emit stable telemetry. diff --git a/cli/azd/extensions/azure.ai.agents/docs/private-networking.md b/cli/azd/extensions/azure.ai.agents/docs/private-networking.md index 3f92ad53d0f..a3ddde8cefb 100644 --- a/cli/azd/extensions/azure.ai.agents/docs/private-networking.md +++ b/cli/azd/extensions/azure.ai.agents/docs/private-networking.md @@ -8,6 +8,7 @@ The `azure.ai.projects` extension owns the project service and the `microsoft.fo When `network:` is present, azd always provisions an **account private endpoint** and disables public data-plane access. Dependent stores (Cosmos DB, AI Search, Storage) stay platform-managed. + ```yaml infra: provider: microsoft.foundry @@ -98,6 +99,7 @@ azd env set AZURE_DNS_SUBSCRIPTION_ID "" Omit `agentSubnet` so the hosted-agent runtime uses a Microsoft-managed network. `peSubnet` is still required: the account data plane stays private behind an account private endpoint in your VNet, reachable from inside the VNet, a peered VNet, or VPN. + ```yaml infra: provider: microsoft.foundry @@ -142,6 +144,7 @@ azd ai agent invoke --new-session "hello" Set `agentSubnet` to inject the hosted-agent runtime into your customer subnet. `agentSubnet` and `peSubnet` must reference the same VNet in v1. + ```yaml services: my-agent: diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go new file mode 100644 index 00000000000..60794b0494d --- /dev/null +++ b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go @@ -0,0 +1,372 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" + "gopkg.in/yaml.v3" +) + +// This file validates the `azure.yaml` examples embedded in this extension's +// markdown docs. Doc snippets used to drift from the code with nothing in CI to +// catch it: the README's migration example once omitted the required agent +// `name`, so copying it failed with "name cannot be empty". See issue #9330. +// +// Two checks run against every fenced YAML block that declares an +// `azure.ai.agent` service: +// +// 1. Resolver — the snippet must survive [AgentDefinitionFromService], the same +// entry point azd uses at deploy time. +// 2. Vocabulary — every property must be one azd actually understands. azd +// ignores unrecognized service properties at runtime (deliberately, for +// forward compatibility), which is how a doc can advertise a setting that +// silently does nothing. Our own docs should not rely on that leniency. + +// docExamplePartialMarker opts a snippet out of the "must fully resolve" check. +// +// Not every example is meant to be complete. The `azure.ai.agent` entries in +// docs/private-networking.md intentionally omit `kind` because the snippet is +// about the project's network block; azd falls back to the on-disk agent.yaml. +// Place this marker on its own line immediately before the opening fence. +const docExamplePartialMarker = "" + +// coreServiceKeys are the `services.` properties that azd core parses into +// typed fields on its own ServiceConfig. Everything else in a service block is +// captured by core's `yaml:",inline"` AdditionalProperties map and handed to the +// extension. Mirrors ServiceConfig in cli/azd/pkg/project/service_config.go. +var coreServiceKeys = []string{ + "apiVersion", "condition", "config", "dist", "docker", "env", "hooks", "host", + "image", "infra", "k8s", "language", "module", "project", "remoteBuild", + "resourceGroup", "resourceName", "uses", +} + +// docExample is a single fenced YAML block extracted from a markdown file. +type docExample struct { + file string + line int + content string + partial bool +} + +// String identifies the snippet in test output as file:line, so a failure points +// straight at the fence that needs fixing. +func (e docExample) String() string { + return fmt.Sprintf("%s:%d", e.file, e.line) +} + +// extensionRoot returns the extension's root directory (the parent of internal/). +func extensionRoot(t *testing.T) string { + t.Helper() + root, err := filepath.Abs(filepath.Join("..", "..")) + require.NoError(t, err) + return root +} + +// extractYAMLExamples returns every fenced YAML block in the markdown source. +// Fences indented inside list items are dedented by the fence's own indent so +// the captured content parses as standalone YAML. +func extractYAMLExamples(file, source string) []docExample { + var examples []docExample + + lines := strings.Split(strings.ReplaceAll(source, "\r\n", "\n"), "\n") + partial := false + + for i := 0; i < len(lines); i++ { + trimmed := strings.TrimSpace(lines[i]) + + if trimmed == docExamplePartialMarker { + partial = true + continue + } + + open := countLeadingBackticks(trimmed) + if open < 3 { + // Only a blank line may sit between the marker and its fence. + if trimmed != "" { + partial = false + } + continue + } + + lang := strings.TrimSpace(trimmed[open:]) + indent := strings.Index(lines[i], "`") + start := i + 1 + + // Per CommonMark, a fence closes only on a run of at least as many + // backticks as opened it, so a longer fence can wrap shorter ones. + var body []string + for i++; i < len(lines) && !closesFence(lines[i], open); i++ { + body = append(body, dedent(lines[i], indent)) + } + + if lang == "yaml" || lang == "yml" { + examples = append(examples, docExample{ + file: file, + line: start, + content: strings.Join(body, "\n"), + partial: partial, + }) + } + partial = false + } + + return examples +} + +// countLeadingBackticks returns the length of the backtick run at the start of line. +func countLeadingBackticks(line string) int { + n := 0 + for n < len(line) && line[n] == '`' { + n++ + } + return n +} + +// closesFence reports whether line is a bare run of at least open backticks. +func closesFence(line string, open int) bool { + trimmed := strings.TrimSpace(line) + n := countLeadingBackticks(trimmed) + return n >= open && n == len(trimmed) +} + +// dedent removes up to n leading spaces from line. +func dedent(line string, n int) string { + for range n { + if !strings.HasPrefix(line, " ") { + break + } + line = line[1:] + } + return line +} + +// agentServiceKeys returns the property names documented for the +// `azure.ai.agent` service block, read from the extension's published schema. +func agentServiceKeys(t *testing.T, root string) []string { + t.Helper() + + raw, err := os.ReadFile(filepath.Join(root, "schemas", "azure.ai.agent.json")) + require.NoError(t, err) + + var schema struct { + Properties map[string]json.RawMessage `json:"properties"` + } + require.NoError(t, json.Unmarshal(raw, &schema)) + require.NotEmpty(t, schema.Properties, "azure.ai.agent.json declares no properties") + + keys := make([]string, 0, len(schema.Properties)) + for k := range schema.Properties { + keys = append(keys, k) + } + return keys +} + +// serviceConfigFromDoc builds the ServiceConfig azd core would hand this +// extension for the given service block, applying core's split: typed fields for +// known keys, AdditionalProperties for everything else. +func serviceConfigFromDoc(t *testing.T, name string, svc map[string]any) *azdext.ServiceConfig { + t.Helper() + + str := func(key string) string { + s, _ := svc[key].(string) + return s + } + + extra := map[string]any{} + for k, v := range svc { + if !slices.Contains(coreServiceKeys, k) { + extra[k] = v + } + } + + props, err := structpb.NewStruct(extra) + require.NoError(t, err) + + out := &azdext.ServiceConfig{ + Name: name, + Host: str("host"), + Language: str("language"), + RelativePath: str("project"), + Image: str("image"), + AdditionalProperties: props, + } + + // The deprecated shape nests the agent definition under `config`. + if cfg, ok := svc["config"].(map[string]any); ok { + legacy, err := structpb.NewStruct(cfg) + require.NoError(t, err) + out.Config = legacy + } + + return out +} + +// agentServicesInExample returns the `azure.ai.agent` service blocks declared by +// the snippet, keyed by service name. +func agentServicesInExample(t *testing.T, e docExample) map[string]map[string]any { + t.Helper() + + var doc struct { + Services map[string]map[string]any `yaml:"services"` + } + require.NoError(t, yaml.Unmarshal([]byte(e.content), &doc), "%s: snippet is not valid YAML", e) + + found := map[string]map[string]any{} + for name, svc := range doc.Services { + if host, _ := svc["host"].(string); host == "azure.ai.agent" { + found[name] = svc + } + } + return found +} + +// TestDocExamplesAreValid runs every `azure.ai.agent` snippet in this +// extension's docs through the resolver azd uses at deploy time, so a doc a user +// copies into azure.yaml cannot drift into a broken state unnoticed. +func TestDocExamplesAreValid(t *testing.T) { + t.Parallel() + + root := extensionRoot(t) + knownKeys := append(agentServiceKeys(t, root), coreServiceKeys...) + + var files []string + require.NoError(t, filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.EqualFold(filepath.Ext(path), ".md") { + files = append(files, path) + } + return nil + })) + + var examples []docExample + for _, path := range files { + source, err := os.ReadFile(path) + require.NoError(t, err) + + rel, err := filepath.Rel(root, path) + require.NoError(t, err) + + examples = append(examples, extractYAMLExamples(filepath.ToSlash(rel), string(source))...) + } + + checked := 0 + for _, e := range examples { + if !strings.Contains(e.content, "azure.ai.agent") { + continue + } + + for name, svc := range agentServicesInExample(t, e) { + checked++ + + t.Run(fmt.Sprintf("%s/%s", e, name), func(t *testing.T) { + t.Parallel() + + cfg := serviceConfigFromDoc(t, name, svc) + _, _, found, _, err := AgentDefinitionFromService(cfg) + + require.NoError(t, err, + "%s: service %q is not a valid agent definition. "+ + "Fix the example so it can be copied into azure.yaml as-is.", e, name) + + if !e.partial { + require.True(t, found, + "%s: service %q did not resolve to an agent definition (is `kind` missing?). "+ + "If the snippet is deliberately incomplete, put %s on the line before the fence.", + e, name, docExamplePartialMarker) + } + + // Guard the silent-no-op class of defect: azd ignores properties + // it does not recognize, so an undocumented key deploys cleanly + // while doing nothing at all. + for key := range definitionProps(svc) { + require.Contains(t, knownKeys, key, + "%s: service %q documents property %q, which azd does not support. "+ + "azd ignores unknown properties, so users copying this get no error and no effect.", + e, name, key) + } + }) + } + } + + require.NotZero(t, checked, "no azure.ai.agent doc examples were found — is the extractor still working?") +} + +// definitionProps returns the properties carrying the agent definition: the +// service block itself, or its `config` child for the deprecated nested shape. +func definitionProps(svc map[string]any) map[string]any { + if cfg, ok := svc["config"].(map[string]any); ok { + if _, nested := cfg["kind"]; nested { + return cfg + } + } + return svc +} + +// TestExtractYAMLExamples covers the extractor itself, since every other +// assertion in this file depends on it finding the right blocks. +func TestExtractYAMLExamples(t *testing.T) { + t.Parallel() + + source := strings.Join([]string{ + "# Doc", + "", + "```yaml", + "services:", + " a:", + " host: azure.ai.agent", + "```", + "", + "```bash", + "not: yaml", + "```", + "", + docExamplePartialMarker, + "```yaml", + "services:", + " b:", + " host: azure.ai.agent", + "```", + "", + "- list item:", + "", + " ```yaml", + " services:", + " c:", + " host: azure.ai.agent", + " ```", + "", + "````markdown", + docExamplePartialMarker, + "```yaml", + "services:", + " d:", + " host: azure.ai.agent", + "```", + "````", + }, "\n") + + got := extractYAMLExamples("doc.md", source) + require.Len(t, got, 3, "bash block must be skipped; fence inside a longer fence must not be extracted") + + require.False(t, got[0].partial) + require.Equal(t, 3, got[0].line) + + require.True(t, got[1].partial, "marker must apply to the fence that follows it") + + require.False(t, got[2].partial, "marker must not leak to a later fence") + require.Equal(t, "services:\n c:\n host: azure.ai.agent", got[2].content, + "indented fences must be dedented") +} From c40566fd9852115f44d29ea2c9e74537a1b5bc2c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:54:15 +0000 Subject: [PATCH 2/5] test(azure.ai.agents): validate doc snippets through core's field shapes and the full schema Co-authored-by: glharper <64209257+glharper@users.noreply.github.com> --- .../internal/project/doc_examples_test.go | 232 ++++++++++++++---- 1 file changed, 179 insertions(+), 53 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go index 60794b0494d..40bd1ace529 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go @@ -51,6 +51,106 @@ var coreServiceKeys = []string{ "resourceGroup", "resourceName", "uses", } +// docSchema is the extension's published JSON Schema for the `azure.ai.agent` +// service block, used to check that every property a doc advertises is one the +// extension actually declares — including properties nested inside documented +// objects, which the non-strict unmarshalling this test guards against would +// otherwise drop silently. +type docSchema struct { + root map[string]any +} + +// loadDocSchema reads schemas/azure.ai.agent.json from the extension root. +func loadDocSchema(t *testing.T, root string) *docSchema { + t.Helper() + + raw, err := os.ReadFile(filepath.Join(root, "schemas", "azure.ai.agent.json")) + require.NoError(t, err) + + var schema map[string]any + require.NoError(t, json.Unmarshal(raw, &schema)) + + s := &docSchema{root: schema} + require.NotEmpty(t, s.properties(schema), "azure.ai.agent.json declares no properties") + return s +} + +// property returns the schema node declared for a top-level service property. +func (s *docSchema) property(key string) (map[string]any, bool) { + node, ok := s.properties(s.root)[key].(map[string]any) + return node, ok +} + +// properties returns the `properties` map of a schema node, resolving nothing. +func (s *docSchema) properties(node map[string]any) map[string]any { + props, _ := node["properties"].(map[string]any) + return props +} + +// resolve follows a local `$ref` (`#/definitions/`) to the node it names. +func (s *docSchema) resolve(node map[string]any) map[string]any { + ref, ok := node["$ref"].(string) + if !ok { + return node + } + + name := strings.TrimPrefix(ref, "#/definitions/") + defs, _ := s.root["definitions"].(map[string]any) + if target, ok := defs[name].(map[string]any); ok { + return s.resolve(target) + } + return node +} + +// checkValue asserts that value contains no property the schema node does not +// declare, recursing through objects and array items. A node that permits +// additional properties (the JSON Schema default) is not narrowed further. +func (s *docSchema) checkValue(t *testing.T, e docExample, name, path string, node map[string]any, value any) { + t.Helper() + + node = s.resolve(node) + + switch v := value.(type) { + case map[string]any: + props := s.properties(node) + additional, declared := node["additionalProperties"] + + for key, child := range v { + childPath := path + "." + key + + if sub, ok := props[key].(map[string]any); ok { + s.checkValue(t, e, name, childPath, sub, child) + continue + } + if sub, ok := additional.(map[string]any); ok { + s.checkValue(t, e, name, childPath, sub, child) + continue + } + if allowed, ok := additional.(bool); !declared || (ok && allowed) { + continue + } + require.Fail(t, "undeclared property", undeclaredPropertyMessage(e, name, childPath)) + } + case []any: + items, ok := node["items"].(map[string]any) + if !ok { + return + } + for i, item := range v { + s.checkValue(t, e, name, fmt.Sprintf("%s[%d]", path, i), items, item) + } + } +} + +// undeclaredPropertyMessage explains why an unsupported property is a defect +// rather than a harmless extra, since azd itself reports nothing. +func undeclaredPropertyMessage(e docExample, name, path string) string { + return fmt.Sprintf( + "%s: service %q documents property %q, which azd does not support. "+ + "azd ignores unknown properties, so users copying this get no error and no effect.", + e, name, path) +} + // docExample is a single fenced YAML block extracted from a markdown file. type docExample struct { file string @@ -151,60 +251,68 @@ func dedent(line string, n int) string { return line } -// agentServiceKeys returns the property names documented for the -// `azure.ai.agent` service block, read from the extension's published schema. -func agentServiceKeys(t *testing.T, root string) []string { - t.Helper() - - raw, err := os.ReadFile(filepath.Join(root, "schemas", "azure.ai.agent.json")) - require.NoError(t, err) - - var schema struct { - Properties map[string]json.RawMessage `json:"properties"` - } - require.NoError(t, json.Unmarshal(raw, &schema)) - require.NotEmpty(t, schema.Properties, "azure.ai.agent.json declares no properties") - - keys := make([]string, 0, len(schema.Properties)) - for k := range schema.Properties { - keys = append(keys, k) - } - return keys +// coreServiceFields mirrors the typed fields azd core parses out of a +// `services.` block, following ServiceConfig in +// cli/azd/pkg/project/service_config.go. Decoding a snippet through it applies +// core's own shape rules — `project: [src]` is a type error for core and must be +// one here too — instead of silently coercing a malformed value to its zero +// value. Everything core does not name lands in AdditionalProperties, exactly as +// core's `yaml:",inline"` field does before the block reaches this extension. +// +// Core's package is mirrored rather than imported: it is not part of this +// module's dependency surface, and pulling it in for a docs test would drag +// azd's provisioning tree behind it. +type coreServiceFields struct { + ResourceGroupName string `yaml:"resourceGroup"` + ResourceName string `yaml:"resourceName"` + ApiVersion string `yaml:"apiVersion"` + RelativePath string `yaml:"project"` + Host string `yaml:"host"` + Language string `yaml:"language"` + OutputPath string `yaml:"dist"` + Image string `yaml:"image"` + Docker map[string]any `yaml:"docker"` + K8s map[string]any `yaml:"k8s"` + Module string `yaml:"module"` + Infra map[string]any `yaml:"infra"` + Hooks map[string]any `yaml:"hooks"` + Uses []string `yaml:"uses"` + Config map[string]any `yaml:"config"` + Environment map[string]string `yaml:"env"` + Condition string `yaml:"condition"` + RemoteBuild *bool `yaml:"remoteBuild"` + AdditionalProperties map[string]any `yaml:",inline"` } // serviceConfigFromDoc builds the ServiceConfig azd core would hand this -// extension for the given service block, applying core's split: typed fields for -// known keys, AdditionalProperties for everything else. -func serviceConfigFromDoc(t *testing.T, name string, svc map[string]any) *azdext.ServiceConfig { +// extension for the given service block, failing the test if any field core +// itself parses has a shape core would reject. +func serviceConfigFromDoc(t *testing.T, e docExample, name string, svc map[string]any) *azdext.ServiceConfig { t.Helper() - str := func(key string) string { - s, _ := svc[key].(string) - return s - } + raw, err := yaml.Marshal(svc) + require.NoError(t, err) - extra := map[string]any{} - for k, v := range svc { - if !slices.Contains(coreServiceKeys, k) { - extra[k] = v - } - } + var core coreServiceFields + require.NoError(t, yaml.Unmarshal(raw, &core), + "%s: service %q cannot be parsed by azd core. "+ + "Fix the example so it can be copied into azure.yaml as-is.", e, name) - props, err := structpb.NewStruct(extra) + props, err := structpb.NewStruct(core.AdditionalProperties) require.NoError(t, err) out := &azdext.ServiceConfig{ Name: name, - Host: str("host"), - Language: str("language"), - RelativePath: str("project"), - Image: str("image"), + Host: core.Host, + Language: core.Language, + RelativePath: core.RelativePath, + Image: core.Image, AdditionalProperties: props, } // The deprecated shape nests the agent definition under `config`. - if cfg, ok := svc["config"].(map[string]any); ok { - legacy, err := structpb.NewStruct(cfg) + if core.Config != nil { + legacy, err := structpb.NewStruct(core.Config) require.NoError(t, err) out.Config = legacy } @@ -238,7 +346,7 @@ func TestDocExamplesAreValid(t *testing.T) { t.Parallel() root := extensionRoot(t) - knownKeys := append(agentServiceKeys(t, root), coreServiceKeys...) + schema := loadDocSchema(t, root) var files []string require.NoError(t, filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { @@ -274,7 +382,7 @@ func TestDocExamplesAreValid(t *testing.T) { t.Run(fmt.Sprintf("%s/%s", e, name), func(t *testing.T) { t.Parallel() - cfg := serviceConfigFromDoc(t, name, svc) + cfg := serviceConfigFromDoc(t, e, name, svc) _, _, found, _, err := AgentDefinitionFromService(cfg) require.NoError(t, err, @@ -291,12 +399,7 @@ func TestDocExamplesAreValid(t *testing.T) { // Guard the silent-no-op class of defect: azd ignores properties // it does not recognize, so an undocumented key deploys cleanly // while doing nothing at all. - for key := range definitionProps(svc) { - require.Contains(t, knownKeys, key, - "%s: service %q documents property %q, which azd does not support. "+ - "azd ignores unknown properties, so users copying this get no error and no effect.", - e, name, key) - } + checkVocabulary(t, e, name, svc, schema) }) } } @@ -304,15 +407,38 @@ func TestDocExamplesAreValid(t *testing.T) { require.NotZero(t, checked, "no azure.ai.agent doc examples were found — is the extractor still working?") } -// definitionProps returns the properties carrying the agent definition: the -// service block itself, or its `config` child for the deprecated nested shape. -func definitionProps(svc map[string]any) map[string]any { +// checkVocabulary asserts that every property the snippet declares is one azd +// understands: a key azd core parses itself, or a property the extension's +// schema declares — recursively, so neither an unsupported sibling of `config` +// in the deprecated shape nor an unsupported key nested inside a documented +// object slips through. +func checkVocabulary(t *testing.T, e docExample, name string, svc map[string]any, schema *docSchema) { + t.Helper() + + for key, value := range svc { + // Core owns the shape of its own keys; coreServiceFields already + // validated them. + if slices.Contains(coreServiceKeys, key) { + continue + } + + prop, ok := schema.property(key) + require.True(t, ok, undeclaredPropertyMessage(e, name, key)) + schema.checkValue(t, e, name, key, prop, value) + } + + // The deprecated shape nests the agent definition under `config`, which core + // passes through untyped — so the extension's schema, not core, owns every + // key inside it. if cfg, ok := svc["config"].(map[string]any); ok { - if _, nested := cfg["kind"]; nested { - return cfg + for key, value := range cfg { + path := "config." + key + + prop, ok := schema.property(key) + require.True(t, ok, undeclaredPropertyMessage(e, name, path)) + schema.checkValue(t, e, name, path, prop, value) } } - return svc } // TestExtractYAMLExamples covers the extractor itself, since every other From ebd7b45bb7ebbf93eaf5a8b6d67a485c5727a375 Mon Sep 17 00:00:00 2001 From: Glenn Harper Date: Tue, 4 Aug 2026 09:37:28 -0400 Subject: [PATCH 3/5] test(azure.ai.agents): name the doc and the fix when a value can't reach structpb The two structpb.NewStruct assertions were the only doc-content-reachable ones without a message, so a value structpb can't represent failed with a bare "proto: invalid type: time.Time" and no hint about what to change. It is reachable from a snippet: an unquoted date under an extension-owned key (metadata: released: 2024-07-18) decodes to time.Time and fails exactly that way. Both assertions now report file:line/service and say to quote ambiguous scalars. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1d5b46e6-fc04-48bb-9a2c-156105966b48 --- .../azure.ai.agents/internal/project/doc_examples_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go index 40bd1ace529..e8086eaf202 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go @@ -299,7 +299,9 @@ func serviceConfigFromDoc(t *testing.T, e docExample, name string, svc map[strin "Fix the example so it can be copied into azure.yaml as-is.", e, name) props, err := structpb.NewStruct(core.AdditionalProperties) - require.NoError(t, err) + require.NoError(t, err, + "%s: service %q has a value that cannot be represented in azure.yaml "+ + "(quote ambiguous scalars such as dates).", e, name) out := &azdext.ServiceConfig{ Name: name, @@ -313,7 +315,9 @@ func serviceConfigFromDoc(t *testing.T, e docExample, name string, svc map[strin // The deprecated shape nests the agent definition under `config`. if core.Config != nil { legacy, err := structpb.NewStruct(core.Config) - require.NoError(t, err) + require.NoError(t, err, + "%s: service %q has a value under `config` that cannot be represented in "+ + "azure.yaml (quote ambiguous scalars such as dates).", e, name) out.Config = legacy } From 3ca644019f02f717c5fea8a3006c5f6548bebcf1 Mon Sep 17 00:00:00 2001 From: Glenn Harper Date: Tue, 4 Aug 2026 10:25:40 -0400 Subject: [PATCH 4/5] docs(agents): mark the config env snippet as a partial example The service-scoped env section added by #9079 shows only where `env:` belongs, so its snippet has no `kind:` and cannot resolve to a full agent definition. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fed9e97b-e79b-4889-ac76-0d9a428599cd --- cli/azd/extensions/azure.ai.agents/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/azd/extensions/azure.ai.agents/README.md b/cli/azd/extensions/azure.ai.agents/README.md index b9c6b46b30a..a5bb790fa13 100644 --- a/cli/azd/extensions/azure.ai.agents/README.md +++ b/cli/azd/extensions/azure.ai.agents/README.md @@ -60,6 +60,7 @@ carries `config: env:` gets a warning naming the affected variables on both Move them up one level to fix it: + ```yaml services: my-agent: From 1e858bcdea97a391015f4f56b1cbd24f8a18a637 Mon Sep 17 00:00:00 2001 From: Glenn Harper Date: Wed, 5 Aug 2026 12:06:22 -0400 Subject: [PATCH 5/5] Validate doc examples against the full schema and core YAML shapes Addresses review on #9368. Compile azure.ai.agent.json and validate the runtime-active inline or legacy config property map, so required, type, enum, pattern, and nested additionalProperties constraints are enforced in addition to the existing friendly vocabulary check. Reject extension properties in the inactive location because azd selects one shape rather than merging them. Replace untyped core service placeholders with strict test-only mirrors of Docker, AKS/Helm/Kustomize, infra/layers/deploymentStacks, and hook YAML. Mirror HooksConfig's mapping-or-list parsing and core's nil-hook validation, and reject nested core-field typos that azd would otherwise ignore. Regression coverage exercises each constraint and field family. The JSON Schema validator was already a transitive dependency; mark it direct now that the doc test imports it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: b4617dfd-adfb-4b30-9222-477a041f8af9 --- cli/azd/extensions/azure.ai.agents/AGENTS.md | 10 +- cli/azd/extensions/azure.ai.agents/go.mod | 2 +- .../internal/project/doc_examples_test.go | 625 +++++++++++++++++- 3 files changed, 607 insertions(+), 30 deletions(-) diff --git a/cli/azd/extensions/azure.ai.agents/AGENTS.md b/cli/azd/extensions/azure.ai.agents/AGENTS.md index 39046d111d4..33cae71ca0b 100644 --- a/cli/azd/extensions/azure.ai.agents/AGENTS.md +++ b/cli/azd/extensions/azure.ai.agents/AGENTS.md @@ -45,9 +45,13 @@ That `replace` points this extension at your local `cli/azd` checkout instead of YAML block declaring an `azure.ai.agent` service must: 1. Resolve through `AgentDefinitionFromService` without error, and -2. Use only properties declared in `schemas/azure.ai.agent.json` or parsed by azd - core. azd ignores unknown service properties at runtime, so an undocumented - key deploys cleanly while doing nothing — the test blocks that in our docs. +2. Satisfy `schemas/azure.ai.agent.json`, including required fields, types, + enums, patterns, and declared properties, and +3. Parse core-owned service fields (`docker`, `k8s`, `infra`, `hooks`, and the + scalar fields) with the same YAML shapes azd core expects. + +azd ignores unknown service properties at runtime, so an undocumented key +deploys cleanly while doing nothing — the test blocks that in our docs. Snippets that are deliberately incomplete (for example, the network examples in `docs/private-networking.md`, which omit `kind` because azd falls back to the diff --git a/cli/azd/extensions/azure.ai.agents/go.mod b/cli/azd/extensions/azure.ai.agents/go.mod index f4f2f359cf9..08d62684277 100644 --- a/cli/azd/extensions/azure.ai.agents/go.mod +++ b/cli/azd/extensions/azure.ai.agents/go.mod @@ -35,6 +35,7 @@ require ( github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 github.com/creack/pty v1.1.24 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 golang.org/x/term v0.44.0 @@ -100,7 +101,6 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/theckman/yacspin v0.13.12 // indirect diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go index e8086eaf202..cd678554ba9 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/doc_examples_test.go @@ -4,6 +4,7 @@ package project import ( + "bytes" "encoding/json" "fmt" "os" @@ -13,6 +14,7 @@ import ( "testing" "github.com/azure/azure-dev/cli/azd/pkg/azdext" + jsonschema "github.com/santhosh-tekuri/jsonschema/v6" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/structpb" "gopkg.in/yaml.v3" @@ -23,15 +25,20 @@ import ( // catch it: the README's migration example once omitted the required agent // `name`, so copying it failed with "name cannot be empty". See issue #9330. // -// Two checks run against every fenced YAML block that declares an +// Three checks run against every fenced YAML block that declares an // `azure.ai.agent` service: // // 1. Resolver — the snippet must survive [AgentDefinitionFromService], the same // entry point azd uses at deploy time. -// 2. Vocabulary — every property must be one azd actually understands. azd -// ignores unrecognized service properties at runtime (deliberately, for -// forward compatibility), which is how a doc can advertise a setting that -// silently does nothing. Our own docs should not rely on that leniency. +// 2. JSON Schema — extension-owned values must satisfy required, type, enum, +// pattern, and additionalProperties constraints. +// 3. Core YAML — core-owned values must have the same shapes azd core parses. +// +// The vocabulary part of check 2 is intentionally stricter than the schema's +// top-level additionalProperties setting. azd ignores unrecognized service +// properties at runtime (deliberately, for forward compatibility), which is how +// a doc can advertise a setting that silently does nothing. Our own docs should +// not rely on that leniency. // docExamplePartialMarker opts a snippet out of the "must fully resolve" check. // @@ -57,7 +64,8 @@ var coreServiceKeys = []string{ // objects, which the non-strict unmarshalling this test guards against would // otherwise drop silently. type docSchema struct { - root map[string]any + root map[string]any + compiled *jsonschema.Schema } // loadDocSchema reads schemas/azure.ai.agent.json from the extension root. @@ -70,11 +78,27 @@ func loadDocSchema(t *testing.T, root string) *docSchema { var schema map[string]any require.NoError(t, json.Unmarshal(raw, &schema)) - s := &docSchema{root: schema} + const resourceURI = "mem://azure.ai.agent.json" + compiler := jsonschema.NewCompiler() + require.NoError(t, compiler.AddResource(resourceURI, schema)) + + compiled, err := compiler.Compile(resourceURI) + require.NoError(t, err) + + s := &docSchema{ + root: schema, + compiled: compiled, + } require.NotEmpty(t, s.properties(schema), "azure.ai.agent.json declares no properties") return s } +// validate applies every JSON Schema constraint, including required fields, +// types, enums, patterns, and additionalProperties. +func (s *docSchema) validate(value map[string]any) error { + return s.compiled.Validate(value) +} + // property returns the schema node declared for a top-level service property. func (s *docSchema) property(key string) (map[string]any, bool) { node, ok := s.properties(s.root)[key].(map[string]any) @@ -251,17 +275,213 @@ func dedent(line string, n int) string { return line } -// coreServiceFields mirrors the typed fields azd core parses out of a +// coreDockerFields mirrors project.DockerProjectOptions without importing the +// full project package into this test-only validator. +type coreDockerFields struct { + Path string `yaml:"path"` + Context string `yaml:"context"` + Platform string `yaml:"platform"` + Target string `yaml:"target"` + Registry string `yaml:"registry"` + Image string `yaml:"image"` + Tag string `yaml:"tag"` + RemoteBuild bool `yaml:"remoteBuild"` + Network string `yaml:"network"` + BuildArgs []string `yaml:"buildArgs"` +} + +// coreK8sFields mirrors the azure.yaml-facing portion of project.AksOptions. +type coreK8sFields struct { + Namespace string `yaml:"namespace"` + DeploymentPath string `yaml:"deploymentPath"` + Ingress coreK8sIngressFields `yaml:"ingress"` + Deployment coreK8sDeploymentFields `yaml:"deployment"` + Service coreK8sServiceFields `yaml:"service"` + Helm *coreHelmFields `yaml:"helm"` + Kustomize *coreKustomizeFields `yaml:"kustomize"` +} + +type coreK8sIngressFields struct { + Name string `yaml:"name"` + RelativePath string `yaml:"relativePath"` +} + +type coreK8sDeploymentFields struct { + Name string `yaml:"name"` +} + +type coreK8sServiceFields struct { + Name string `yaml:"name"` +} + +type coreHelmFields struct { + Repositories []*coreHelmRepositoryFields `yaml:"repositories"` + Releases []*coreHelmReleaseFields `yaml:"releases"` +} + +type coreHelmRepositoryFields struct { + Name string `yaml:"name"` + URL string `yaml:"url"` +} + +type coreHelmReleaseFields struct { + Name string `yaml:"name"` + Chart string `yaml:"chart"` + Version string `yaml:"version"` + Namespace string `yaml:"namespace"` + Values string `yaml:"values"` +} + +type coreKustomizeFields struct { + Directory string `yaml:"dir"` + Edits []string `yaml:"edits"` + Env map[string]string `yaml:"env"` +} + +type coreDeploymentStacksFields struct { + ActionOnUnmanage *coreActionOnUnmanageFields `yaml:"actionOnUnmanage"` + DenySettings *coreDenySettingsFields `yaml:"denySettings"` +} + +type coreActionOnUnmanageFields struct { + Resources string `yaml:"resources"` + ResourceGroups string `yaml:"resourceGroups"` + ManagementGroups string `yaml:"managementGroups"` +} + +type coreDenySettingsFields struct { + Mode string `yaml:"mode"` + ApplyToChildScopes *bool `yaml:"applyToChildScopes"` + ExcludedActions []string `yaml:"excludedActions"` + ExcludedPrincipals []string `yaml:"excludedPrincipals"` +} + +// coreHookFields mirrors the YAML-facing fields of ext.HookConfig. +type coreHookFields struct { + Name string `yaml:",omitempty"` + Kind string `yaml:"kind"` + Shell string `yaml:"shell"` + Dir string `yaml:"dir"` + Run string `yaml:"run"` + ContinueOnError bool `yaml:"continueOnError"` + Interactive bool `yaml:"interactive"` + Windows *coreHookFields `yaml:"windows"` + Posix *coreHookFields `yaml:"posix"` + Secrets map[string]string `yaml:"secrets"` + Config map[string]any `yaml:"config"` +} + +// coreHooksFields mirrors ext.HooksConfig.UnmarshalYAML: each hook is either a +// mapping or a sequence of mappings. Scalars are rejected instead of being +// silently accepted by a map[string]any placeholder. +type coreHooksFields map[string][]*coreHookFields + +// strictYAMLUnmarshal is deliberately stricter than core's forward-compatible +// decoder. Runtime must allow future extension fields, but this extension's own +// docs must not publish a misspelled core field that azd silently ignores. +func strictYAMLUnmarshal(data []byte, value any) error { + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + return decoder.Decode(value) +} + +func (h *coreHooksFields) UnmarshalYAML(unmarshal func(any) error) error { + var raw map[string]any + if err := unmarshal(&raw); err != nil { + return err + } + + result := make(coreHooksFields, len(raw)) + for name, value := range raw { + switch value.(type) { + case nil: + result[name] = []*coreHookFields{nil} + case map[string]any: + encoded, err := yaml.Marshal(value) + if err != nil { + return err + } + var hook coreHookFields + if err := strictYAMLUnmarshal(encoded, &hook); err != nil { + return fmt.Errorf("failed to unmarshal hook %q: %w", name, err) + } + result[name] = []*coreHookFields{&hook} + case []any: + encoded, err := yaml.Marshal(value) + if err != nil { + return err + } + var hooks []*coreHookFields + if err := strictYAMLUnmarshal(encoded, &hooks); err != nil { + return fmt.Errorf("failed to unmarshal hook %q: %w", name, err) + } + result[name] = hooks + default: + return fmt.Errorf( + "failed to unmarshal hook %q: expected mapping or sequence, got %T", + name, value, + ) + } + } + + *h = result + return nil +} + +func validateCoreHooks(hooks coreHooksFields) error { + for name, list := range hooks { + if list == nil { + return fmt.Errorf( + "hook %q has an empty definition; expected properties such as run or shell", + name, + ) + } + for i, hook := range list { + if hook == nil { + return fmt.Errorf( + "hook %q entry %d has an empty definition; expected properties such as run or shell", + name, i+1, + ) + } + } + } + return nil +} + +// coreInfraFields mirrors the azure.yaml-facing portion of +// provisioning.Options. +type coreInfraFields struct { + Provider string `yaml:"provider"` + Path string `yaml:"path"` + Module string `yaml:"module"` + Name string `yaml:"name"` + Hooks coreHooksFields `yaml:"hooks"` + DeploymentStacks *coreDeploymentStacksFields `yaml:"deploymentStacks"` + Config map[string]any `yaml:"config"` + DependsOn []string `yaml:"dependsOn"` + Layers []coreInfraFields `yaml:"layers"` +} + +func validateCoreInfraFields(infra coreInfraFields) error { + if err := validateCoreHooks(infra.Hooks); err != nil { + return fmt.Errorf("infra: %w", err) + } + for i, layer := range infra.Layers { + if err := validateCoreInfraFields(layer); err != nil { + return fmt.Errorf("infra.layers[%d]: %w", i, err) + } + } + return nil +} + +// coreServiceFields mirrors the scalar fields azd core parses out of a // `services.` block, following ServiceConfig in -// cli/azd/pkg/project/service_config.go. Decoding a snippet through it applies -// core's own shape rules — `project: [src]` is a type error for core and must be -// one here too — instead of silently coercing a malformed value to its zero -// value. Everything core does not name lands in AdditionalProperties, exactly as -// core's `yaml:",inline"` field does before the block reaches this extension. +// cli/azd/pkg/project/service_config.go. Complex core-owned fields mirror their +// exact YAML shapes and unmarshalling rules — for example, a scalar hook value +// must fail exactly as HooksConfig rejects it at runtime. // -// Core's package is mirrored rather than imported: it is not part of this -// module's dependency surface, and pulling it in for a docs test would drag -// azd's provisioning tree behind it. +// Everything core does not name lands in AdditionalProperties, exactly as +// core's `yaml:",inline"` field does before the block reaches this extension. type coreServiceFields struct { ResourceGroupName string `yaml:"resourceGroup"` ResourceName string `yaml:"resourceName"` @@ -271,11 +491,11 @@ type coreServiceFields struct { Language string `yaml:"language"` OutputPath string `yaml:"dist"` Image string `yaml:"image"` - Docker map[string]any `yaml:"docker"` - K8s map[string]any `yaml:"k8s"` + Docker coreDockerFields `yaml:"docker"` + K8s coreK8sFields `yaml:"k8s"` Module string `yaml:"module"` - Infra map[string]any `yaml:"infra"` - Hooks map[string]any `yaml:"hooks"` + Infra coreInfraFields `yaml:"infra"` + Hooks coreHooksFields `yaml:"hooks"` Uses []string `yaml:"uses"` Config map[string]any `yaml:"config"` Environment map[string]string `yaml:"env"` @@ -284,17 +504,35 @@ type coreServiceFields struct { AdditionalProperties map[string]any `yaml:",inline"` } +// decodeCoreServiceFields applies the YAML types and custom unmarshallers azd +// core uses for its part of a service block. +func decodeCoreServiceFields(svc map[string]any) (coreServiceFields, error) { + raw, err := yaml.Marshal(svc) + if err != nil { + return coreServiceFields{}, err + } + + var core coreServiceFields + if err := strictYAMLUnmarshal(raw, &core); err != nil { + return coreServiceFields{}, err + } + if err := validateCoreHooks(core.Hooks); err != nil { + return coreServiceFields{}, err + } + if err := validateCoreInfraFields(core.Infra); err != nil { + return coreServiceFields{}, err + } + return core, nil +} + // serviceConfigFromDoc builds the ServiceConfig azd core would hand this // extension for the given service block, failing the test if any field core // itself parses has a shape core would reject. func serviceConfigFromDoc(t *testing.T, e docExample, name string, svc map[string]any) *azdext.ServiceConfig { t.Helper() - raw, err := yaml.Marshal(svc) - require.NoError(t, err) - - var core coreServiceFields - require.NoError(t, yaml.Unmarshal(raw, &core), + core, err := decodeCoreServiceFields(svc) + require.NoError(t, err, "%s: service %q cannot be parsed by azd core. "+ "Fix the example so it can be copied into azure.yaml as-is.", e, name) @@ -324,6 +562,135 @@ func serviceConfigFromDoc(t *testing.T, e docExample, name string, svc map[strin return out } +func TestDecodeCoreServiceFieldsRejectsMalformedValues(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + content string + }{ + { + name: "docker must be an object", + content: `host: azure.ai.agent +docker: invalid`, + }, + { + name: "docker nested fields keep their core types", + content: `host: azure.ai.agent +docker: + remoteBuild: sometimes`, + }, + { + name: "docker rejects unknown fields", + content: `host: azure.ai.agent +docker: + buildArgz: [FOO=bar]`, + }, + { + name: "k8s must be an object", + content: `host: azure.ai.agent +k8s: invalid`, + }, + { + name: "k8s nested fields keep their core types", + content: `host: azure.ai.agent +k8s: + helm: + repositories: invalid`, + }, + { + name: "k8s rejects unknown fields", + content: `host: azure.ai.agent +k8s: + namespaze: default`, + }, + { + name: "infra must be an object", + content: `host: azure.ai.agent +infra: invalid`, + }, + { + name: "infra nested fields keep their core types", + content: `host: azure.ai.agent +infra: + layers: invalid`, + }, + { + name: "infra rejects unknown fields", + content: `host: azure.ai.agent +infra: + providerz: bicep`, + }, + { + name: "deployment stacks keep their core types", + content: `host: azure.ai.agent +infra: + deploymentStacks: + denySettings: invalid`, + }, + { + name: "deployment stacks reject unknown fields", + content: `host: azure.ai.agent +infra: + deploymentStacks: + denySettingz: + mode: none`, + }, + { + name: "hook must be an object or list", + content: `host: azure.ai.agent +hooks: + preprovision: 1`, + }, + { + name: "hook fields keep their core types", + content: `host: azure.ai.agent +hooks: + preprovision: + run: [not, a, string]`, + }, + { + name: "hook rejects unknown fields", + content: `host: azure.ai.agent +hooks: + preprovision: + continueOnErrror: true`, + }, + { + name: "empty hook definition", + content: `host: azure.ai.agent +hooks: + preprovision:`, + }, + { + name: "empty hook list entry", + content: `host: azure.ai.agent +hooks: + preprovision: + - null`, + }, + { + name: "empty infra hook definition", + content: `host: azure.ai.agent +infra: + hooks: + preprovision:`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + var svc map[string]any + require.NoError(t, yaml.Unmarshal([]byte(test.content), &svc)) + + _, err := decodeCoreServiceFields(svc) + require.Error(t, err) + }) + } +} + // agentServicesInExample returns the `azure.ai.agent` service blocks declared by // the snippet, keyed by service name. func agentServicesInExample(t *testing.T, e docExample) map[string]map[string]any { @@ -343,6 +710,174 @@ func agentServicesInExample(t *testing.T, e docExample) map[string]map[string]an return found } +func TestDocSchemaValidatesConstraints(t *testing.T) { + t.Parallel() + + schema := loadDocSchema(t, extensionRoot(t)) + type fixture struct { + value map[string]any + deployment map[string]any + sku map[string]any + } + validDeployment := func() fixture { + sku := map[string]any{ + "name": "GlobalStandard", + "capacity": 10, + } + deployment := map[string]any{ + "name": "gpt-4o", + "model": map[string]any{ + "name": "gpt-4o", + "format": "OpenAI", + "version": "2024-08-06", + }, + "sku": sku, + } + return fixture{ + value: map[string]any{ + "deployments": []any{ + deployment, + }, + }, + deployment: deployment, + sku: sku, + } + } + + tests := []struct { + name string + mutate func(*fixture) + wantErr bool + }{ + {name: "valid deployment"}, + { + name: "required model", + mutate: func(value *fixture) { + delete(value.deployment, "model") + }, + wantErr: true, + }, + { + name: "required sku", + mutate: func(value *fixture) { + delete(value.deployment, "sku") + }, + wantErr: true, + }, + { + name: "capacity type", + mutate: func(value *fixture) { + value.sku["capacity"] = "ten" + }, + wantErr: true, + }, + { + name: "kind enum", + mutate: func(value *fixture) { + value.value["kind"] = "serverless" + }, + wantErr: true, + }, + { + name: "connection name pattern", + mutate: func(value *fixture) { + value.value["connections"] = []any{ + map[string]any{ + "name": "!", + "category": "CustomKeys", + "target": "https://example.test", + "authType": "None", + }, + } + }, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + value := validDeployment() + if test.mutate != nil { + test.mutate(&value) + } + + err := schema.validate(value.value) + if test.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestActiveDocAgentConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + inline map[string]any + config map[string]any + want map[string]any + wantLocation string + wantErr string + }{ + { + name: "inline only", + inline: map[string]any{"kind": "hosted", "name": "agent"}, + want: map[string]any{"kind": "hosted", "name": "agent"}, + wantLocation: "inline", + }, + { + name: "config only", + config: map[string]any{"kind": "hosted", "name": "agent"}, + want: map[string]any{"kind": "hosted", "name": "agent"}, + wantLocation: "config", + }, + { + name: "no extension properties", + want: map[string]any{}, + wantLocation: "inline", + }, + { + name: "inline definition makes config inactive", + inline: map[string]any{"kind": "hosted", "name": "agent"}, + config: map[string]any{"container": map[string]any{}}, + wantErr: "deprecated config properties are ignored", + }, + { + name: "config definition makes inline inactive", + inline: map[string]any{"container": map[string]any{}}, + config: map[string]any{"kind": "hosted", "name": "agent"}, + wantErr: "inline properties are ignored", + }, + { + name: "inline properties win when neither location has kind", + inline: map[string]any{"container": map[string]any{}}, + config: map[string]any{"deployments": []any{}}, + wantErr: "deprecated config properties are ignored", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + got, location, err := activeDocAgentConfig(test.inline, test.config) + if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + return + } + + require.NoError(t, err) + require.Equal(t, test.want, got) + require.Equal(t, test.wantLocation, location) + }) + } +} + // TestDocExamplesAreValid runs every `azure.ai.agent` snippet in this // extension's docs through the resolver azd uses at deploy time, so a doc a user // copies into azure.yaml cannot drift into a broken state unnoticed. @@ -419,6 +954,7 @@ func TestDocExamplesAreValid(t *testing.T) { func checkVocabulary(t *testing.T, e docExample, name string, svc map[string]any, schema *docSchema) { t.Helper() + inline := map[string]any{} for key, value := range svc { // Core owns the shape of its own keys; coreServiceFields already // validated them. @@ -429,12 +965,14 @@ func checkVocabulary(t *testing.T, e docExample, name string, svc map[string]any prop, ok := schema.property(key) require.True(t, ok, undeclaredPropertyMessage(e, name, key)) schema.checkValue(t, e, name, key, prop, value) + inline[key] = value } // The deprecated shape nests the agent definition under `config`, which core // passes through untyped — so the extension's schema, not core, owns every // key inside it. - if cfg, ok := svc["config"].(map[string]any); ok { + cfg, _ := svc["config"].(map[string]any) + if cfg != nil { for key, value := range cfg { path := "config." + key @@ -443,6 +981,41 @@ func checkVocabulary(t *testing.T, e docExample, name string, svc map[string]any schema.checkValue(t, e, name, path, prop, value) } } + + active, location, err := activeDocAgentConfig(inline, cfg) + require.NoError(t, err, "%s: service %q contains ignored agent properties", e, name) + require.NoError(t, schema.validate(active), + "%s: service %q %s properties do not satisfy schemas/azure.ai.agent.json. "+ + "Fix the example so it can be copied into azure.yaml as-is.", e, name, location) +} + +// activeDocAgentConfig mirrors ServiceConfigProps: inline extension properties +// win unless they omit kind and the deprecated config block declares it. Since +// the locations are never merged, documenting both would make one silently +// ineffective and is therefore rejected. +func activeDocAgentConfig(inline, config map[string]any) (map[string]any, string, error) { + if len(inline) == 0 && len(config) == 0 { + return map[string]any{}, "inline", nil + } + if len(inline) == 0 { + return config, "config", nil + } + if len(config) == 0 { + return inline, "inline", nil + } + + inlineKind, _ := inline["kind"].(string) + configKind, _ := config["kind"].(string) + if inlineKind == "" && configKind != "" { + return nil, "", fmt.Errorf( + "inline properties are ignored because deprecated config declares the active agent definition; " + + "move them under config or migrate the whole definition inline", + ) + } + return nil, "", fmt.Errorf( + "deprecated config properties are ignored because the inline agent definition is active; " + + "remove config or migrate all of its properties inline", + ) } // TestExtractYAMLExamples covers the extractor itself, since every other