From e9d0a40d7c799e8a97983f43548bf4a3c26d000b Mon Sep 17 00:00:00 2001 From: Osher-Elhadad Date: Wed, 3 Jun 2026 17:27:08 +0300 Subject: [PATCH] feat(authbridge): add SPARC pre-tool reflection plugin, service, and demo SPARC checks that an agent's proposed tool call is grounded in the conversation and the available tool specs before it runs, catching hallucinated or ungrounded arguments such as an invented transaction id. - authlib/plugins/sparc: the Go plugin. It collects SPARC's three inputs from the inference and mcp parsers, enforces per format (an MCP tool result in mcp mode, a rewritten completion in inference mode), and records verdicts as normal pipeline invocations plus a structured session event. - sparc-service: a small FastAPI wrapper over ALTK's SPARCReflectionComponent. The provider is set via environment: watsonx by default, with ollama, openai, azure, and a generic litellm option for anything else LiteLLM supports. - demos/finance-sparc: a regular MCP finance agent where SPARC catches a hallucinated transaction id and the agent asks the user to confirm it before issuing the refund. - docs/sparc-plugin.md and the demo README. Signed-off-by: Osher-Elhadad --- authbridge/CLAUDE.md | 2 +- authbridge/authlib/plugins/sparc/collect.go | 165 ++++++ authbridge/authlib/plugins/sparc/plugin.go | 513 ++++++++++++++++++ .../authlib/plugins/sparc/plugin_test.go | 346 ++++++++++++ authbridge/authlib/plugins/sparc/reflector.go | 129 +++++ .../authlib/plugins/sparc/reflector_test.go | 92 ++++ authbridge/authlib/plugins/sparc/respond.go | 123 +++++ authbridge/cmd/authbridge-envoy/main.go | 1 + authbridge/cmd/authbridge-proxy/main.go | 1 + authbridge/demos/README.md | 1 + authbridge/demos/finance-sparc/.gitignore | 1 + authbridge/demos/finance-sparc/Makefile | 213 ++++++++ authbridge/demos/finance-sparc/README.md | 183 +++++++ .../finance-sparc/finance-agent/Dockerfile | 12 + .../demos/finance-sparc/finance-agent/main.go | 471 ++++++++++++++++ .../finance-sparc/finance-mcp/Dockerfile | 11 + .../demos/finance-sparc/finance-mcp/main.go | 220 ++++++++ authbridge/demos/finance-sparc/go.mod | 3 + authbridge/demos/finance-sparc/k8s/agent.yaml | 121 +++++ .../demos/finance-sparc/k8s/finance-mcp.yaml | 63 +++ .../demos/finance-sparc/k8s/ollama.yaml | 95 ++++ .../demos/finance-sparc/k8s/sparc-patch.yaml | 53 ++ .../finance-sparc/k8s/sparc-service.yaml | 82 +++ .../demos/finance-sparc/scripts/drive-demo.sh | 70 +++ .../demos/finance-sparc/scripts/host-setup.sh | 53 ++ .../finance-sparc/scripts/launch-abctl.sh | 20 + .../scripts/patch-sparc-config.sh | 111 ++++ .../finance-sparc/scripts/pipeline-merge.py | 49 ++ .../scripts/setup_keycloak_finance.py | 97 ++++ .../finance-sparc/scripts/show-verdicts.py | 26 + .../scripts/stage-ollama-model.sh | 34 ++ authbridge/docs/sparc-plugin.md | 165 ++++++ authbridge/sparc-service/.dockerignore | 7 + authbridge/sparc-service/Dockerfile | 24 + authbridge/sparc-service/README.md | 136 +++++ authbridge/sparc-service/pyproject.toml | 34 ++ .../sparc-service/sparc_service/__init__.py | 16 + .../sparc-service/sparc_service/__main__.py | 16 + authbridge/sparc-service/sparc_service/api.py | 70 +++ .../sparc-service/sparc_service/engine.py | 136 +++++ .../sparc-service/sparc_service/models.py | 60 ++ .../sparc-service/sparc_service/providers.py | 116 ++++ .../sparc-service/sparc_service/settings.py | 175 ++++++ authbridge/sparc-service/tests/conftest.py | 68 +++ authbridge/sparc-service/tests/test_api.py | 110 ++++ authbridge/sparc-service/tests/test_engine.py | 115 ++++ .../sparc-service/tests/test_providers.py | 52 ++ .../sparc-service/tests/test_settings.py | 105 ++++ .../tests/test_watsonx_integration.py | 86 +++ 49 files changed, 4851 insertions(+), 1 deletion(-) create mode 100644 authbridge/authlib/plugins/sparc/collect.go create mode 100644 authbridge/authlib/plugins/sparc/plugin.go create mode 100644 authbridge/authlib/plugins/sparc/plugin_test.go create mode 100644 authbridge/authlib/plugins/sparc/reflector.go create mode 100644 authbridge/authlib/plugins/sparc/reflector_test.go create mode 100644 authbridge/authlib/plugins/sparc/respond.go create mode 100644 authbridge/demos/finance-sparc/.gitignore create mode 100644 authbridge/demos/finance-sparc/Makefile create mode 100644 authbridge/demos/finance-sparc/README.md create mode 100644 authbridge/demos/finance-sparc/finance-agent/Dockerfile create mode 100644 authbridge/demos/finance-sparc/finance-agent/main.go create mode 100644 authbridge/demos/finance-sparc/finance-mcp/Dockerfile create mode 100644 authbridge/demos/finance-sparc/finance-mcp/main.go create mode 100644 authbridge/demos/finance-sparc/go.mod create mode 100644 authbridge/demos/finance-sparc/k8s/agent.yaml create mode 100644 authbridge/demos/finance-sparc/k8s/finance-mcp.yaml create mode 100644 authbridge/demos/finance-sparc/k8s/ollama.yaml create mode 100644 authbridge/demos/finance-sparc/k8s/sparc-patch.yaml create mode 100644 authbridge/demos/finance-sparc/k8s/sparc-service.yaml create mode 100644 authbridge/demos/finance-sparc/scripts/drive-demo.sh create mode 100644 authbridge/demos/finance-sparc/scripts/host-setup.sh create mode 100644 authbridge/demos/finance-sparc/scripts/launch-abctl.sh create mode 100644 authbridge/demos/finance-sparc/scripts/patch-sparc-config.sh create mode 100644 authbridge/demos/finance-sparc/scripts/pipeline-merge.py create mode 100644 authbridge/demos/finance-sparc/scripts/setup_keycloak_finance.py create mode 100644 authbridge/demos/finance-sparc/scripts/show-verdicts.py create mode 100644 authbridge/demos/finance-sparc/scripts/stage-ollama-model.sh create mode 100644 authbridge/docs/sparc-plugin.md create mode 100644 authbridge/sparc-service/.dockerignore create mode 100644 authbridge/sparc-service/Dockerfile create mode 100644 authbridge/sparc-service/README.md create mode 100644 authbridge/sparc-service/pyproject.toml create mode 100644 authbridge/sparc-service/sparc_service/__init__.py create mode 100644 authbridge/sparc-service/sparc_service/__main__.py create mode 100644 authbridge/sparc-service/sparc_service/api.py create mode 100644 authbridge/sparc-service/sparc_service/engine.py create mode 100644 authbridge/sparc-service/sparc_service/models.py create mode 100644 authbridge/sparc-service/sparc_service/providers.py create mode 100644 authbridge/sparc-service/sparc_service/settings.py create mode 100644 authbridge/sparc-service/tests/conftest.py create mode 100644 authbridge/sparc-service/tests/test_api.py create mode 100644 authbridge/sparc-service/tests/test_engine.py create mode 100644 authbridge/sparc-service/tests/test_providers.py create mode 100644 authbridge/sparc-service/tests/test_settings.py create mode 100644 authbridge/sparc-service/tests/test_watsonx_integration.py diff --git a/authbridge/CLAUDE.md b/authbridge/CLAUDE.md index cb67b20e8..ad52f7f5f 100644 --- a/authbridge/CLAUDE.md +++ b/authbridge/CLAUDE.md @@ -11,7 +11,7 @@ binaries with shared auth logic in `authlib/`: - `cmd/authbridge-proxy/` — proxy-sidecar mode (default). HTTP forward + reverse proxies. Full plugin set (jwt-validation, token-exchange, a2a-parser, - mcp-parser, inference-parser, ibac). + mcp-parser, inference-parser, ibac, sparc). - `cmd/authbridge-envoy/` — envoy-sidecar mode. ext_proc gRPC server hooked into Envoy. Full plugin set. - `cmd/authbridge-lite/` — proxy-sidecar mode, lite plugin set (auth gates diff --git a/authbridge/authlib/plugins/sparc/collect.go b/authbridge/authlib/plugins/sparc/collect.go new file mode 100644 index 000000000..5c66c29de --- /dev/null +++ b/authbridge/authlib/plugins/sparc/collect.go @@ -0,0 +1,165 @@ +package sparc + +import ( + "encoding/json" + "fmt" + "path" + "strings" + + "github.com/kagenti/kagenti-extensions/authbridge/authlib/pipeline" +) + +// correlateInferenceContext pulls the conversation (incl. system prompt) and +// the tool inventory from the session's most recent outbound inference request +// — i.e. exactly what the agent sent to its LLM. Used in mcp mode, where the +// concrete tool call comes from the MCP request but the context must be sourced +// from the agent's LLM call. Returns ok=false when no inference event exists +// (no inference-parser, or session tracking off) so the caller can skip rather +// than reflect on partial data. +func correlateInferenceContext(pctx *pipeline.Context) (messages, toolSpecs []map[string]any, ok bool) { + if pctx.Session == nil { + return nil, nil, false + } + infs := pctx.Session.InferenceRequests() + if len(infs) == 0 { + return nil, nil, false + } + inf := infs[len(infs)-1].Inference + if inf == nil || len(inf.Messages) == 0 { + return nil, nil, false + } + messages, toolSpecs = inferenceMessagesAndTools(inf) + return messages, toolSpecs, true +} + +// inferenceInputs extracts messages + tool specs directly from the inference +// extension on the current request/response (inference mode — all inputs are +// co-located, no correlation needed). +func inferenceInputs(inf *pipeline.InferenceExtension) (messages, toolSpecs []map[string]any, ok bool) { + if inf == nil || len(inf.Messages) == 0 { + return nil, nil, false + } + messages, toolSpecs = inferenceMessagesAndTools(inf) + return messages, toolSpecs, true +} + +func inferenceMessagesAndTools(inf *pipeline.InferenceExtension) (messages, toolSpecs []map[string]any) { + for _, m := range inf.Messages { + messages = append(messages, map[string]any{"role": m.Role, "content": m.Content}) + } + for _, t := range inf.Tools { + toolSpecs = append(toolSpecs, map[string]any{ + "type": "function", + "function": map[string]any{ + "name": t.Name, + "description": t.Description, + "parameters": t.Parameters, + }, + }) + } + return messages, toolSpecs +} + +// buildToolCall renders the MCP tools/call as an OpenAI-style tool call (the +// shape SPARC expects). Arguments are serialized to a JSON string. +func buildToolCall(mcp *pipeline.MCPExtension, toolName string) map[string]any { + return openAIToolCall(fmt.Sprintf("%v", mcpRPCID(mcp)), toolName, extractMCPToolArgs(mcp)) +} + +// openAIToolCall builds the canonical OpenAI function tool-call object. args is +// a JSON string (defaulted to "{}"). +func openAIToolCall(id, name, args string) map[string]any { + if args == "" { + args = "{}" + } + return map[string]any{ + "id": id, + "type": "function", + "function": map[string]any{"name": name, "arguments": args}, + } +} + +func extractMCPToolName(mcp *pipeline.MCPExtension) string { + if mcp == nil || mcp.Method != "tools/call" { + return "" + } + if name, ok := mcp.Params["name"].(string); ok { + return name + } + return "" +} + +func extractMCPToolArgs(mcp *pipeline.MCPExtension) string { + if mcp == nil || mcp.Method != "tools/call" { + return "" + } + args, ok := mcp.Params["arguments"] + if !ok { + return "" + } + b, err := json.Marshal(args) + if err != nil { + return "" + } + return string(b) +} + +func mcpRPCID(mcp *pipeline.MCPExtension) any { + if mcp == nil { + return nil + } + return mcp.RPCID +} + +func sessionID(pctx *pipeline.Context) string { + if pctx.Session != nil { + return pctx.Session.ID + } + return "" +} + +func firstExplanation(verdict ReflectVerdict) string { + for _, iss := range verdict.Issues { + if iss.Explanation != "" { + return iss.Explanation + } + } + return "tool call not grounded in the conversation" +} + +func scoreString(score *float64) string { + if score == nil { + return "" + } + return fmt.Sprintf("%.2f", *score) +} + +// anyGlobMatch reports whether name matches any path.Match glob in patterns. +func anyGlobMatch(patterns []string, name string) bool { + for _, p := range patterns { + if matched, _ := path.Match(p, name); matched { + return true + } + } + return false +} + +// matchesAnyHost reports whether host matches any glob (path.Match), port stripped. +func matchesAnyHost(patterns []string, host string) bool { + if host == "" { + return false + } + if i := strings.IndexByte(host, ':'); i >= 0 { + host = host[:i] + } + return anyGlobMatch(patterns, host) +} + +// preview truncates s to n runes, appending an ellipsis when truncated. +func preview(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} diff --git a/authbridge/authlib/plugins/sparc/plugin.go b/authbridge/authlib/plugins/sparc/plugin.go new file mode 100644 index 000000000..012ded3f9 --- /dev/null +++ b/authbridge/authlib/plugins/sparc/plugin.go @@ -0,0 +1,513 @@ +// Package sparc implements a pre-tool reflection plugin for authbridge. +// +// Before an agent's proposed tool call executes, the plugin asks an external +// SPARC reflection service (the agent-lifecycle-toolkit / ALTK SPARC component, +// served over HTTP) whether the call is *grounded* in the conversation and the +// available tool specifications. SPARC catches hallucinated / ungrounded +// arguments (e.g. an invented transaction id) and inappropriate tool selection. +// +// Generic by design — works for ANY kagenti agent. SPARC's three inputs are +// collected from exactly what the agent produces, with no bespoke per-agent +// wiring: +// +// - conversation history (INCLUDING the system prompt) — from the agent's +// LLM call captured by inference-parser (pctx.Extensions.Inference.Messages, +// which preserves every role verbatim); +// - tool specifications in OpenAI function-calling format — from the same +// inference call (pctx.Extensions.Inference.Tools); +// - the proposed tool call in OpenAI format — from the outbound MCP +// tools/call (mcp mode) or from the LLM response (inference mode). +// +// Enforcement is format-aware (the verdict is returned to the agent in the +// shape it expects): +// +// - enforcement: "mcp" (default; the kagenti norm) — gate the outbound MCP +// tools/call. On a reflected reject, return SPARC's clarification as a +// JSON-RPC MCP tool *result*; the agent's MCP client consumes it like any +// tool output and asks the user for the missing detail. Robust to LLM +// streaming (reads the actual tool call, not the streamed response). +// - enforcement: "inference" — gate the agent's LLM response (where all three +// inputs are co-located). On a reflected reject, rewrite the completion so +// the assistant turn carries SPARC's clarification (tool_calls dropped). +// For agents that don't route tools through MCP. +// +// All enforcement policy lives here; the service only returns SPARC's verdict. +package sparc + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "path" + "strings" + "time" + + "github.com/kagenti/kagenti-extensions/authbridge/authlib/bypass" + "github.com/kagenti/kagenti-extensions/authbridge/authlib/pipeline" + "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins" +) + +// enforcement modes. +const ( + EnforcementMCP = "mcp" // gate the outbound MCP tools/call (default) + EnforcementInference = "inference" // gate the agent's LLM response (non-MCP agents) +) + +// on_reject_action values: what to do when SPARC returns "reject". +const ( + // OnRejectObserve records the reject and lets the call through (shadow mode). + OnRejectObserve = "observe" + // OnRejectReflect returns SPARC's reflection to the agent (MCP tool result + // in mcp mode, or a rewritten LLM response in inference mode) so the agent + // transparently re-asks the user. This is "reflection mode". + OnRejectReflect = "reflect" + // OnRejectDeny hard-blocks the call. + OnRejectDeny = "deny" +) + +// fail_policy values: what to do when SPARC is unavailable or returns "error". +const ( + FailOpen = "open" // allow + record (default — SPARC is a quality gate, not auth) + FailClosed = "closed" // block +) + +// sparcConfig is the plugin's local config schema. Field tags drive both +// runtime decoding (json) and operator-facing schema introspection +// (description / required / default / enum), matching ibac and the auth plugins. +// See authbridge/docs/sparc-plugin.md for prose. +type sparcConfig struct { + ReflectorEndpoint string `json:"reflector_endpoint" required:"true" description:"Base URL of the SPARC reflection service. The plugin POSTs to {endpoint}/reflect."` + + ReflectorBearer string `json:"reflector_bearer" description:"Optional bearer token for the reflection service. Empty for in-cluster unauthenticated calls."` + + Enforcement string `json:"enforcement" description:"Where to gate and how to return the verdict: mcp=gate the MCP tools/call, return an MCP tool result; inference=gate the LLM response, rewrite the completion." default:"mcp" enum:"mcp,inference"` + + Track string `json:"track" description:"SPARC reflection track." default:"fast_track" enum:"fast_track,slow_track,syntax,spec_free,transformations_only"` + + TimeoutMs int `json:"timeout_ms" description:"Per-call reflection timeout. SPARC LLM calls take seconds. Validation rejects values below 100." default:"30000"` + + OnRejectAction string `json:"on_reject_action" description:"Behavior on a SPARC reject: observe=log only; reflect=return SPARC's clarification to the agent; deny=hard block." default:"reflect" enum:"observe,reflect,deny"` + + DenyScoreThreshold float64 `json:"deny_score_threshold" description:"When SPARC's grounding score (0=worst..1=best) is <= this, escalate any reject to a hard deny regardless of on_reject_action. 0 disables escalation." default:"0"` + + FailPolicy string `json:"fail_policy" description:"On SPARC unavailable or decision=error: open=allow and record; closed=block." default:"open" enum:"open,closed"` + + SkipTools []string `json:"skip_tools" description:"Tool-name globs (path.Match) to NOT reflect on, e.g. trivial read-only tools. Evaluated before reflect_tools."` + + ReflectTools []string `json:"reflect_tools" description:"If non-empty, ONLY reflect on tools whose name matches one of these globs; all others are skipped."` + + BypassHosts []string `json:"bypass_hosts" description:"Host globs (path.Match) skipped without reflecting. Defaults include keycloak / spire / otel."` + + BypassPaths []string `json:"bypass_paths" description:"URL path globs skipped without reflecting. Defaults: /.well-known/* /healthz /readyz /livez."` +} + +// defaultBypassHosts mirrors ibac's conservative starting set (kept local so a +// future ibac tweak can't silently change sparc behavior). +var defaultBypassHosts = []string{ + "keycloak.*", "keycloak", "spire-server.*", "spire-agent.*", + "otel-collector.*", "jaeger.*", "prometheus.*", +} + +var defaultBypassPaths = []string{"/healthz", "/readyz", "/livez", "/.well-known/*"} + +func (c *sparcConfig) applyDefaults() { + if c.Enforcement == "" { + c.Enforcement = EnforcementMCP + } + if c.Track == "" { + c.Track = "fast_track" + } + if c.TimeoutMs == 0 { + c.TimeoutMs = 30000 + } + if c.OnRejectAction == "" { + c.OnRejectAction = OnRejectReflect + } + if c.FailPolicy == "" { + c.FailPolicy = FailOpen + } + if len(c.BypassHosts) == 0 { + c.BypassHosts = defaultBypassHosts + } + if len(c.BypassPaths) == 0 { + c.BypassPaths = defaultBypassPaths + } +} + +func (c *sparcConfig) validate() error { + if c.ReflectorEndpoint == "" { + return errors.New("reflector_endpoint is required") + } + if c.TimeoutMs < 100 { + return fmt.Errorf("timeout_ms must be at least 100, got %d", c.TimeoutMs) + } + switch c.Enforcement { + case EnforcementMCP, EnforcementInference: + default: + return fmt.Errorf("enforcement must be mcp or inference, got %q", c.Enforcement) + } + switch c.OnRejectAction { + case OnRejectObserve, OnRejectReflect, OnRejectDeny: + default: + return fmt.Errorf("on_reject_action must be observe|reflect|deny, got %q", c.OnRejectAction) + } + switch c.FailPolicy { + case FailOpen, FailClosed: + default: + return fmt.Errorf("fail_policy must be open or closed, got %q", c.FailPolicy) + } + if c.DenyScoreThreshold < 0 || c.DenyScoreThreshold > 1 { + return fmt.Errorf("deny_score_threshold must be in [0,1], got %v", c.DenyScoreThreshold) + } + for _, g := range []struct { + label string + pats []string + }{{"bypass_hosts", c.BypassHosts}, {"bypass_paths", c.BypassPaths}, {"skip_tools", c.SkipTools}, {"reflect_tools", c.ReflectTools}} { + for _, p := range g.pats { + if _, err := path.Match(p, ""); err != nil { + return fmt.Errorf("invalid %s pattern %q: %w", g.label, p, err) + } + } + } + for _, p := range c.BypassHosts { + if t := strings.TrimSpace(p); t == "" || t == "*" { + return fmt.Errorf("bypass_hosts pattern %q matches everything; remove sparc from the pipeline instead", p) + } + } + for _, p := range c.BypassPaths { + if t := strings.TrimSpace(p); t == "" || t == "*" || t == "/*" { + return fmt.Errorf("bypass_paths pattern %q matches everything; remove sparc from the pipeline instead", p) + } + } + return nil +} + +// SPARC reflects on proposed tool calls and enforces the configured policy. +type SPARC struct { + cfg sparcConfig + reflector Reflector + bypassPaths *bypass.Matcher + bypassHosts []string + timeout time.Duration +} + +// NewSPARC constructs an unconfigured plugin. Configure must be called first. +func NewSPARC() *SPARC { return &SPARC{} } + +func init() { + plugins.RegisterPlugin("sparc", func() pipeline.Plugin { return NewSPARC() }) +} + +func (p *SPARC) Name() string { return "sparc" } + +func (p *SPARC) Capabilities() pipeline.PluginCapabilities { + return pipeline.PluginCapabilities{ + // Needs a parser to supply SPARC's inputs. inference-parser provides the + // conversation + tool specs (both modes); mcp-parser provides the tool + // call (mcp mode). RequiresAny is a static "at least one" check; the + // per-mode runtime requirements are validated/handled below. + RequiresAny: []string{"inference-parser", "mcp-parser"}, + ReadsBody: true, + WritesBody: true, // MCP result (mcp mode) / completion rewrite (inference mode) + Description: "SPARC pre-tool reflection: blocks ungrounded/hallucinated tool calls.", + } +} + +func (p *SPARC) ConfigSchema() []pipeline.FieldSchema { + return pipeline.SchemaOf(sparcConfig{}) +} + +func (p *SPARC) Configure(raw json.RawMessage) error { + var c sparcConfig + if len(raw) > 0 { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&c); err != nil { + return fmt.Errorf("sparc config: %w", err) + } + } + c.applyDefaults() + if err := c.validate(); err != nil { + return fmt.Errorf("sparc config: %w", err) + } + p.cfg = c + + matcher, err := bypass.NewMatcher(c.BypassPaths) + if err != nil { + return fmt.Errorf("sparc bypass_paths: %w", err) + } + p.bypassPaths = matcher + p.bypassHosts = c.BypassHosts + p.timeout = time.Duration(c.TimeoutMs) * time.Millisecond + p.reflector = newHTTPReflector(c.ReflectorEndpoint, c.ReflectorBearer, p.timeout) + return nil +} + +// OnRequest handles mcp mode (gate the outbound MCP tools/call). +func (p *SPARC) OnRequest(ctx context.Context, pctx *pipeline.Context) pipeline.Action { + if p.reflector == nil { + return pipeline.DenyStatus(503, "sparc.unconfigured", "sparc plugin not configured") + } + if p.cfg.Enforcement != EnforcementMCP { + return cont() + } + if pctx.Headers.Get(sentinelHeader) == "1" { + return cont() + } + if p.bypassPaths.Match(pctx.Path) { + pctx.Skip("path_bypass") + return cont() + } + if matchesAnyHost(p.bypassHosts, pctx.Host) { + pctx.Skip("host_bypass") + return cont() + } + anyAction, anyBypass := pctx.Classification() + if anyBypass { + pctx.Skip("protocol_mechanics") + return cont() + } + if !anyAction { + return cont() + } + + toolName := extractMCPToolName(pctx.Extensions.MCP) + if toolName == "" { + pctx.Skip("not_tool_call") + return cont() + } + if p.toolSkipped(toolName) { + pctx.Skip("tool_skipped") + return cont() + } + + messages, toolSpecs, ok := correlateInferenceContext(pctx) + if !ok { + // No conversation/tool context to ground against. This is a missing + // prerequisite (no inference-parser event / session tracking off), not + // a verdict — apply fail_policy so a misconfigured pipeline doesn't + // silently run unprotected. + return p.handleNoContext(pctx, toolName, EnforcementMCP) + } + in := ReflectInput{ + Messages: messages, + ToolSpecs: toolSpecs, + ToolCalls: []map[string]any{buildToolCall(pctx.Extensions.MCP, toolName)}, + SessionID: sessionID(pctx), + Track: p.cfg.Track, + } + + verdict, err := p.reflector.Reflect(ctx, in) + if err != nil { + return p.handleUnavailable(pctx, toolName, err, EnforcementMCP) + } + p.emitEvent(pctx, toolName, verdict) + + switch verdict.Decision { + case DecisionApprove: + p.recordAllow(pctx, toolName, verdict, EnforcementMCP) + return cont() + case DecisionReject: + return p.handleReject(pctx, toolName, verdict, EnforcementMCP) + default: + return p.handleUnavailable(pctx, toolName, + fmt.Errorf("%w: sparc decision=%q", ErrReflectorUnavailable, verdict.Decision), EnforcementMCP) + } +} + +// OnResponse handles inference mode (gate the agent's LLM response). +func (p *SPARC) OnResponse(ctx context.Context, pctx *pipeline.Context) pipeline.Action { + if p.cfg.Enforcement != EnforcementInference { + return cont() + } + inf := pctx.Extensions.Inference + if inf == nil || len(inf.ToolCalls) == 0 { + return cont() // not an LLM tool-call response (e.g. streamed or plain text) + } + tc := inf.ToolCalls[0] + if tc.Name == "" { + return cont() + } + if p.toolSkipped(tc.Name) { + pctx.Skip("tool_skipped") + return cont() + } + + messages, toolSpecs, ok := inferenceInputs(inf) + if !ok { + return p.handleNoContext(pctx, tc.Name, EnforcementInference) + } + in := ReflectInput{ + Messages: messages, + ToolSpecs: toolSpecs, + ToolCalls: []map[string]any{openAIToolCall(tc.ID, tc.Name, tc.Arguments)}, + SessionID: sessionID(pctx), + Track: p.cfg.Track, + } + + verdict, err := p.reflector.Reflect(ctx, in) + if err != nil { + return p.handleUnavailable(pctx, tc.Name, err, EnforcementInference) + } + p.emitEvent(pctx, tc.Name, verdict) + + switch verdict.Decision { + case DecisionApprove: + p.recordAllow(pctx, tc.Name, verdict, EnforcementInference) + return cont() + case DecisionReject: + return p.handleReject(pctx, tc.Name, verdict, EnforcementInference) + default: + return p.handleUnavailable(pctx, tc.Name, + fmt.Errorf("%w: sparc decision=%q", ErrReflectorUnavailable, verdict.Decision), EnforcementInference) + } +} + +// handleReject applies on_reject_action with score-driven escalation. The +// returned action is format-aware per enforcement mode. +func (p *SPARC) handleReject(pctx *pipeline.Context, toolName string, verdict ReflectVerdict, mode string) pipeline.Action { + action := p.cfg.OnRejectAction + escalated := false + if p.cfg.DenyScoreThreshold > 0 && verdict.OverallAvgScore != nil && + *verdict.OverallAvgScore <= p.cfg.DenyScoreThreshold { + action = OnRejectDeny + escalated = true + } + details := map[string]string{ + "tool": toolName, "score": scoreString(verdict.OverallAvgScore), + "sparc": firstExplanation(verdict), "action": action, + "escalated": fmt.Sprintf("%t", escalated), + } + + switch action { + case OnRejectObserve: + details["mode"] = "observe" + pctx.Record(inv(pipeline.ActionObserve, mode, "reject_observed", details)) + return cont() + case OnRejectReflect: + pctx.Record(inv(pipeline.ActionModify, mode, "reflected", details)) + if mode == EnforcementInference { + rewriteInferenceResponse(pctx, buildClarificationText(verdict)) + return cont() + } + return mcpResultAction(mcpRPCID(pctx.Extensions.MCP), buildClarificationText(verdict)) + default: // OnRejectDeny + pctx.Record(inv(pipeline.ActionDeny, mode, "blocked", details)) + reason := firstExplanation(verdict) + if mode == EnforcementInference { + rewriteInferenceResponse(pctx, "Tool call blocked by policy: "+reason) + return cont() + } + return pipeline.DenyStatus(403, "sparc.blocked", reason) + } +} + +// handleUnavailable applies fail_policy when SPARC can't be reached or errored. +func (p *SPARC) handleUnavailable(pctx *pipeline.Context, toolName string, err error, mode string) pipeline.Action { + errPreview := preview(err.Error(), 240) + details := map[string]string{"tool": toolName, "error": errPreview} + if p.cfg.FailPolicy == FailOpen { + details["mode"] = "fail_open" + pctx.Record(inv(pipeline.ActionObserve, mode, "reflector_unavailable", details)) + slog.Warn("sparc: reflector unavailable, failing open", "host", pctx.Host, "error", errPreview) + return cont() + } + pctx.Record(inv(pipeline.ActionDeny, mode, "reflector_unavailable", details)) + slog.Warn("sparc: reflector unavailable, failing closed", "host", pctx.Host, "error", errPreview) + if mode == EnforcementInference { + rewriteInferenceResponse(pctx, "Tool call blocked: reflection service unavailable.") + return cont() + } + return pipeline.DenyStatus(503, "sparc.reflector_unavailable", errPreview) +} + +// handleNoContext applies fail_policy when the conversation/tool context needed +// to ground a tool call is missing (no inference-parser event, or session +// tracking off). fail_open skips and lets the call through — recorded so the gap +// is visible in the session timeline; fail_closed blocks rather than letting an +// unverifiable call past. Default-open behavior is unchanged from a plain skip. +func (p *SPARC) handleNoContext(pctx *pipeline.Context, toolName, mode string) pipeline.Action { + if p.cfg.FailPolicy == FailOpen { + pctx.Skip("no_inference_context") + return cont() + } + pctx.Record(inv(pipeline.ActionDeny, mode, "no_inference_context", map[string]string{"tool": toolName})) + slog.Warn("sparc: no inference context to ground against, failing closed", "host", pctx.Host, "tool", toolName) + if mode == EnforcementInference { + rewriteInferenceResponse(pctx, "Tool call blocked: no conversation context available to verify it.") + return cont() + } + return pipeline.DenyStatus(503, "sparc.no_inference_context", "no conversation context to verify the tool call") +} + +func (p *SPARC) recordAllow(pctx *pipeline.Context, toolName string, verdict ReflectVerdict, mode string) { + pctx.Record(inv(pipeline.ActionAllow, mode, "grounded", map[string]string{ + "tool": toolName, "score": scoreString(verdict.OverallAvgScore), + })) +} + +// emitEvent publishes the full SPARC verdict via the plugin-event escape-hatch +// so abctl / session consumers can render the structured reflection. +func (p *SPARC) emitEvent(pctx *pipeline.Context, toolName string, verdict ReflectVerdict) { + if pctx.Extensions.Custom == nil { + pctx.Extensions.Custom = map[string]any{} + } + pctx.Extensions.Custom["sparc"+pipeline.PluginEventSuffix] = sparcEvent{ + Tool: toolName, + Decision: verdict.Decision, + Score: verdict.OverallAvgScore, + Track: p.cfg.Track, + Enforcement: p.cfg.Enforcement, + Issues: verdict.Issues, + ExecutionMs: verdict.ExecutionTimeMs, + } +} + +// toolSkipped reports whether a tool is excluded from reflection by config. +func (p *SPARC) toolSkipped(name string) bool { + if anyGlobMatch(p.cfg.SkipTools, name) { + return true + } + if len(p.cfg.ReflectTools) > 0 && !anyGlobMatch(p.cfg.ReflectTools, name) { + return true + } + return false +} + +// sparcEvent is the structured reflection record surfaced to session consumers. +type sparcEvent struct { + Tool string `json:"tool"` + Decision string `json:"decision"` + Score *float64 `json:"score,omitempty"` + Track string `json:"track"` + Enforcement string `json:"enforcement"` + Issues []ReflectIssue `json:"issues,omitempty"` + ExecutionMs float64 `json:"executionMs,omitempty"` +} + +func cont() pipeline.Action { return pipeline.Action{Type: pipeline.Continue} } + +func inv(action pipeline.InvocationAction, mode, reason string, details map[string]string) pipeline.Invocation { + // mcp enforcement runs in OnRequest (request phase); inference enforcement + // runs in OnResponse (response phase). Tag the invocation accordingly so it + // lands in the right session-event snapshot. + phase := pipeline.InvocationPhaseRequest + if mode == EnforcementInference { + phase = pipeline.InvocationPhaseResponse + } + if mode != "" { + details["enforcement"] = mode + } + return pipeline.Invocation{ + Action: action, Phase: phase, Reason: reason, Details: details, + } +} + +// Compile-time interface checks. +var ( + _ pipeline.Plugin = (*SPARC)(nil) + _ pipeline.Configurable = (*SPARC)(nil) +) diff --git a/authbridge/authlib/plugins/sparc/plugin_test.go b/authbridge/authlib/plugins/sparc/plugin_test.go new file mode 100644 index 000000000..ee8414fbe --- /dev/null +++ b/authbridge/authlib/plugins/sparc/plugin_test.go @@ -0,0 +1,346 @@ +package sparc + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/kagenti/kagenti-extensions/authbridge/authlib/pipeline" +) + +type fakeReflector struct { + verdict ReflectVerdict + err error + gotIn ReflectInput + calls int +} + +func (f *fakeReflector) Reflect(_ context.Context, in ReflectInput) (ReflectVerdict, error) { + f.calls++ + f.gotIn = in + return f.verdict, f.err +} + +func configured(t *testing.T, cfgJSON string, fr *fakeReflector) *SPARC { + t.Helper() + p := NewSPARC() + if cfgJSON == "" { + cfgJSON = `{"reflector_endpoint":"http://sparc.invalid","timeout_ms":1000}` + } + if err := p.Configure(json.RawMessage(cfgJSON)); err != nil { + t.Fatalf("Configure: %v", err) + } + if fr != nil { + p.reflector = fr + } + return p +} + +func invokeReq(p pipeline.Plugin, pctx *pipeline.Context) pipeline.Action { + pctx.SetCurrentPlugin(p.Name(), pipeline.InvocationPhaseRequest) + defer pctx.ClearCurrentPlugin() + return p.OnRequest(context.Background(), pctx) +} + +func invokeResp(p pipeline.Plugin, pctx *pipeline.Context) pipeline.Action { + pctx.SetCurrentPlugin(p.Name(), pipeline.InvocationPhaseResponse) + defer pctx.ClearCurrentPlugin() + return p.OnResponse(context.Background(), pctx) +} + +// inferenceEvent builds a session inference request event (the generic source +// of conversation + tool specs). +func inferenceEvent() pipeline.SessionEvent { + return pipeline.SessionEvent{ + Direction: pipeline.Outbound, Phase: pipeline.SessionRequest, + Inference: &pipeline.InferenceExtension{ + Model: "llama3.2:3b", + Messages: []pipeline.InferenceMessage{ + {Role: "system", Content: "You are a finance assistant."}, + {Role: "user", Content: "Refund my duplicate charge."}, + }, + Tools: []pipeline.InferenceTool{ + {Name: "issue_refund", Description: "Issue a refund", Parameters: map[string]any{"type": "object"}}, + }, + }, + } +} + +// mcpPctx builds an outbound pctx for mcp mode: an action-classified MCP +// tools/call plus a session carrying one inference request event. +func mcpPctx() *pipeline.Context { + pctx := &pipeline.Context{ + Direction: pipeline.Outbound, Method: "POST", Scheme: "http", + Host: "finance-mcp.team1.svc", Path: "/mcp", Headers: http.Header{}, + Session: &pipeline.SessionView{ID: "s1", Events: []pipeline.SessionEvent{inferenceEvent()}}, + } + pctx.Extensions.MCP = &pipeline.MCPExtension{ + Method: "tools/call", RPCID: "rpc-1", IsAction: true, + Params: map[string]any{"name": "issue_refund", "arguments": map[string]any{"transaction_id": "TX9999"}}, + } + return pctx +} + +// inferencePctx builds a pctx for inference mode: the LLM response carries a +// proposed tool call plus the request messages+tools. +func inferencePctx() *pipeline.Context { + pctx := &pipeline.Context{ + Direction: pipeline.Outbound, Method: "POST", Scheme: "http", + Host: "ollama", Path: "/v1/chat/completions", Headers: http.Header{}, + ResponseBody: []byte(`{"id":"x","object":"chat.completion","model":"m","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function","function":{"name":"issue_refund","arguments":"{\"transaction_id\":\"TX9999\"}"}}]},"finish_reason":"tool_calls"}]}`), + } + inf := inferenceEvent().Inference + inf.ToolCalls = []pipeline.InferenceToolCall{{ID: "c1", Name: "issue_refund", Arguments: `{"transaction_id":"TX9999"}`}} + pctx.Extensions.Inference = inf + return pctx +} + +func outInvs(pctx *pipeline.Context) []pipeline.Invocation { + if pctx.Extensions.Invocations == nil { + return nil + } + return pctx.Extensions.Invocations.Outbound +} + +// --- Configure --- + +func TestConfigure_RequiresEndpoint(t *testing.T) { + if err := NewSPARC().Configure(json.RawMessage(`{}`)); err == nil { + t.Fatal("expected error when reflector_endpoint missing") + } +} + +func TestConfigure_Defaults(t *testing.T) { + p := configured(t, `{"reflector_endpoint":"http://x"}`, nil) + if p.cfg.Enforcement != EnforcementMCP || p.cfg.Track != "fast_track" || + p.cfg.OnRejectAction != OnRejectReflect || p.cfg.FailPolicy != FailOpen || p.cfg.TimeoutMs != 30000 { + t.Fatalf("defaults not applied: %+v", p.cfg) + } +} + +func TestConfigure_RejectsBadEnums(t *testing.T) { + for _, bad := range []string{ + `{"reflector_endpoint":"http://x","enforcement":"nope"}`, + `{"reflector_endpoint":"http://x","on_reject_action":"nope"}`, + `{"reflector_endpoint":"http://x","fail_policy":"maybe"}`, + `{"reflector_endpoint":"http://x","deny_score_threshold":9}`, + `{"reflector_endpoint":"http://x","bogus":1}`, + } { + if err := NewSPARC().Configure(json.RawMessage(bad)); err == nil { + t.Errorf("expected error for %s", bad) + } + } +} + +// --- mcp mode --- + +func TestMCP_ApproveContinues(t *testing.T) { + score := 4.5 + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionApprove, OverallAvgScore: &score}} + p := configured(t, "", fr) + pctx := mcpPctx() + if act := invokeReq(p, pctx); act.Type != pipeline.Continue { + t.Fatalf("approve should Continue, got %+v", act) + } + // generic collection: system prompt + tools + tool_call all forwarded + if len(fr.gotIn.Messages) != 2 || fr.gotIn.Messages[0]["role"] != "system" { + t.Errorf("expected system+user messages, got %+v", fr.gotIn.Messages) + } + if len(fr.gotIn.ToolSpecs) != 1 || len(fr.gotIn.ToolCalls) != 1 { + t.Errorf("expected tool_specs + tool_call, got %+v", fr.gotIn) + } + if fn := fr.gotIn.ToolCalls[0]["function"].(map[string]any); fn["name"] != "issue_refund" { + t.Errorf("tool_call name = %v", fn["name"]) + } + if i := outInvs(pctx); len(i) != 1 || i[0].Action != pipeline.ActionAllow { + t.Errorf("expected one allow, got %+v", i) + } +} + +func TestMCP_RejectReflects(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject, Issues: []ReflectIssue{{Explanation: "TX9999 not grounded"}}}} + p := configured(t, `{"reflector_endpoint":"http://x","on_reject_action":"reflect"}`, fr) + pctx := mcpPctx() + act := invokeReq(p, pctx) + if act.Type != pipeline.Reject || act.Violation == nil || act.Violation.Status != 200 { + t.Fatalf("reflect should be Reject w/ 200 MCP result, got %+v", act) + } + var env map[string]any + if err := json.Unmarshal(act.Violation.Body, &env); err != nil || env["result"] == nil { + t.Fatalf("body is not an MCP result: %v / %v", err, env) + } + if i := outInvs(pctx); len(i) != 1 || i[0].Action != pipeline.ActionModify { + t.Errorf("expected modify, got %+v", i) + } + // structured event emitted + if _, ok := pctx.Extensions.Custom["sparc"+pipeline.PluginEventSuffix]; !ok { + t.Errorf("expected sparc event in Custom") + } +} + +func TestMCP_RejectObserveContinues(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject, Issues: []ReflectIssue{{Explanation: "x"}}}} + p := configured(t, `{"reflector_endpoint":"http://x","on_reject_action":"observe"}`, fr) + pctx := mcpPctx() + if act := invokeReq(p, pctx); act.Type != pipeline.Continue { + t.Fatalf("observe should Continue, got %+v", act) + } + if i := outInvs(pctx); len(i) != 1 || i[0].Action != pipeline.ActionObserve { + t.Errorf("expected observe, got %+v", i) + } +} + +func TestMCP_RejectDeny(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject, Issues: []ReflectIssue{{Explanation: "blocked"}}}} + p := configured(t, `{"reflector_endpoint":"http://x","on_reject_action":"deny"}`, fr) + act := invokeReq(p, mcpPctx()) + if act.Type != pipeline.Reject || act.Violation == nil || act.Violation.Code != "sparc.blocked" { + t.Fatalf("deny should Reject sparc.blocked, got %+v", act) + } +} + +func TestMCP_ScoreEscalatesToDeny(t *testing.T) { + low := 0.5 + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject, OverallAvgScore: &low, Issues: []ReflectIssue{{Explanation: "severe"}}}} + p := configured(t, `{"reflector_endpoint":"http://x","on_reject_action":"reflect","deny_score_threshold":0.7}`, fr) + act := invokeReq(p, mcpPctx()) + if act.Type != pipeline.Reject || act.Violation == nil || act.Violation.Code != "sparc.blocked" { + t.Fatalf("low score should escalate to deny, got %+v", act) + } +} + +func TestMCP_SkipTools(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject}} + p := configured(t, `{"reflector_endpoint":"http://x","skip_tools":["issue_*"]}`, fr) + if act := invokeReq(p, mcpPctx()); act.Type != pipeline.Continue { + t.Fatalf("skipped tool should Continue, got %+v", act) + } + if fr.calls != 0 { + t.Errorf("reflector should not be called for skipped tool") + } +} + +func TestMCP_ReflectToolsAllowlist(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionApprove}} + p := configured(t, `{"reflector_endpoint":"http://x","reflect_tools":["get_*"]}`, fr) + // issue_refund not in allow-list → skipped + if act := invokeReq(p, mcpPctx()); act.Type != pipeline.Continue || fr.calls != 0 { + t.Fatalf("non-allowlisted tool should be skipped, got act=%+v calls=%d", act, fr.calls) + } +} + +func TestMCP_NoInferenceContextFailsOpen(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject}} + p := configured(t, "", fr) // default fail_policy: open + pctx := mcpPctx() + pctx.Session = &pipeline.SessionView{ID: "s1"} // no inference event + if act := invokeReq(p, pctx); act.Type != pipeline.Continue { + t.Fatalf("no context (fail_open) should Continue, got %+v", act) + } + if fr.calls != 0 { + t.Errorf("reflector should not be called without context") + } +} + +func TestMCP_NoInferenceContextFailsClosed(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject}} + p := configured(t, `{"reflector_endpoint":"http://x","fail_policy":"closed"}`, fr) + pctx := mcpPctx() + pctx.Session = &pipeline.SessionView{ID: "s1"} // no inference event + act := invokeReq(p, pctx) + if act.Type != pipeline.Reject || act.Violation == nil || act.Violation.Status != 503 { + t.Fatalf("no context (fail_closed) should 503, got %+v", act) + } + if fr.calls != 0 { + t.Errorf("reflector should not be called without context") + } +} + +func TestMCP_NotAToolCallSkips(t *testing.T) { + fr := &fakeReflector{} + p := configured(t, "", fr) + pctx := mcpPctx() + pctx.Extensions.MCP.Method = "prompts/get" + if act := invokeReq(p, pctx); act.Type != pipeline.Continue || fr.calls != 0 { + t.Fatalf("non tools/call should skip, got act=%+v calls=%d", act, fr.calls) + } +} + +func TestMCP_Reentrancy(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject}} + p := configured(t, "", fr) + pctx := mcpPctx() + pctx.Headers.Set(sentinelHeader, "1") + if act := invokeReq(p, pctx); act.Type != pipeline.Continue || fr.calls != 0 { + t.Fatalf("reentrant call should Continue without reflecting") + } +} + +func TestMCP_FailOpenAndClosed(t *testing.T) { + frOpen := &fakeReflector{err: ErrReflectorUnavailable} + if act := invokeReq(configured(t, `{"reflector_endpoint":"http://x","fail_policy":"open"}`, frOpen), mcpPctx()); act.Type != pipeline.Continue { + t.Fatalf("fail_open should Continue, got %+v", act) + } + frClosed := &fakeReflector{err: ErrReflectorUnavailable} + act := invokeReq(configured(t, `{"reflector_endpoint":"http://x","fail_policy":"closed"}`, frClosed), mcpPctx()) + if act.Type != pipeline.Reject || act.Violation.Status != 503 { + t.Fatalf("fail_closed should 503, got %+v", act) + } +} + +// --- inference mode --- + +func TestInference_RejectRewritesResponse(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject, Issues: []ReflectIssue{{Explanation: "ungrounded id"}}}} + p := configured(t, `{"reflector_endpoint":"http://x","enforcement":"inference","on_reject_action":"reflect"}`, fr) + pctx := inferencePctx() + if act := invokeResp(p, pctx); act.Type != pipeline.Continue { + t.Fatalf("inference reject should Continue (body rewritten), got %+v", act) + } + // response body rewritten: tool_calls dropped, clarification in content + var out map[string]any + if err := json.Unmarshal(pctx.ResponseBody, &out); err != nil { + t.Fatalf("rewritten body invalid: %v", err) + } + msg := out["choices"].([]any)[0].(map[string]any)["message"].(map[string]any) + if _, hasTC := msg["tool_calls"]; hasTC { + t.Errorf("tool_calls should be dropped, got %v", msg) + } + if c, _ := msg["content"].(string); c == "" { + t.Errorf("clarification content missing") + } + // generic collection: all 3 inputs came straight from the inference ext + if len(fr.gotIn.Messages) != 2 || len(fr.gotIn.ToolSpecs) != 1 || len(fr.gotIn.ToolCalls) != 1 { + t.Errorf("expected 3 inputs co-located, got %+v", fr.gotIn) + } +} + +func TestInference_ApproveLeavesResponse(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionApprove}} + p := configured(t, `{"reflector_endpoint":"http://x","enforcement":"inference"}`, fr) + pctx := inferencePctx() + orig := string(pctx.ResponseBody) + invokeResp(p, pctx) + if string(pctx.ResponseBody) != orig { + t.Errorf("approve should not modify the response") + } +} + +func TestInference_MCPModeOnResponseIsNoop(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject}} + p := configured(t, "", fr) // mcp mode + if act := invokeResp(p, inferencePctx()); act.Type != pipeline.Continue || fr.calls != 0 { + t.Fatalf("mcp-mode OnResponse should be a no-op") + } +} + +func TestCapabilities(t *testing.T) { + caps := NewSPARC().Capabilities() + if !caps.WritesBody || !caps.ReadsBody { + t.Error("expected ReadsBody+WritesBody") + } + if len(caps.RequiresAny) == 0 { + t.Error("expected RequiresAny parsers") + } +} diff --git a/authbridge/authlib/plugins/sparc/reflector.go b/authbridge/authlib/plugins/sparc/reflector.go new file mode 100644 index 000000000..221923e7f --- /dev/null +++ b/authbridge/authlib/plugins/sparc/reflector.go @@ -0,0 +1,129 @@ +package sparc + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" +) + +// Reflector evaluates whether a proposed tool call is grounded in the +// conversation and tool specifications. The concrete implementation +// (httpReflector) calls the out-of-process SPARC reflection service; tests +// inject a fake. +// +// A returned error means the reflection could not be obtained (transport +// failure, timeout, non-2xx). It is distinct from a successful reflection +// whose Decision is "error" (SPARC ran but its own LLM/pipeline failed) — +// the plugin maps both to the configured fail policy, but keeps them +// distinguishable in session records. +type Reflector interface { + Reflect(ctx context.Context, in ReflectInput) (ReflectVerdict, error) +} + +// ReflectInput is SPARC's native input: the conversation, the available +// tools, and the proposed tool call(s) to vet. +type ReflectInput struct { + Messages []map[string]any `json:"messages"` + ToolSpecs []map[string]any `json:"tool_specs"` + ToolCalls []map[string]any `json:"tool_calls"` + SessionID string `json:"session_id,omitempty"` + Track string `json:"track,omitempty"` +} + +// ReflectIssue mirrors one SPARC issue (PyPI agent-lifecycle-toolkit 0.10.1 +// shape: issue_type / metric_name / explanation / correction). +type ReflectIssue struct { + IssueType string `json:"issue_type"` + MetricName string `json:"metric_name"` + Explanation string `json:"explanation"` + Correction any `json:"correction"` +} + +// ReflectVerdict is SPARC's decision plus its supporting issues and score. +type ReflectVerdict struct { + Decision string `json:"decision"` // approve | reject | error + Issues []ReflectIssue `json:"issues"` + OverallAvgScore *float64 `json:"overall_avg_score"` // 0=worst .. 1=best; nil when unavailable + ExecutionTimeMs float64 `json:"execution_time_ms"` +} + +// SPARC decision values. +const ( + DecisionApprove = "approve" + DecisionReject = "reject" + DecisionError = "error" +) + +// ErrReflectorUnavailable is returned (wrapped) when the SPARC service could +// not be reached or returned a non-2xx status — an availability problem, +// distinct from a "reject" verdict. +var ErrReflectorUnavailable = errors.New("sparc reflector unavailable") + +// sentinelHeader marks the plugin's own outbound HTTP call so that, if it +// ever loops back through the listener, OnRequest short-circuits instead of +// recursing (the same defense-in-depth pattern ibac's judge uses). +const sentinelHeader = "X-SPARC-Reflect" + +// httpReflector POSTs a reflection request to {endpoint}/reflect. +// +// The call is made with a standalone http.Client that does NOT route back +// through the authbridge listener, structurally preventing reentrancy. The +// sentinel header is belt-and-suspenders. +type httpReflector struct { + url string + bearer string + client *http.Client + timeout time.Duration +} + +func newHTTPReflector(endpoint, bearer string, timeout time.Duration) *httpReflector { + return &httpReflector{ + url: endpoint + "/reflect", + bearer: bearer, + client: &http.Client{Timeout: timeout}, + timeout: timeout, + } +} + +func (r *httpReflector) Reflect(ctx context.Context, in ReflectInput) (ReflectVerdict, error) { + body, err := json.Marshal(in) + if err != nil { + return ReflectVerdict{}, fmt.Errorf("marshal reflect request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.url, bytes.NewReader(body)) + if err != nil { + return ReflectVerdict{}, fmt.Errorf("%w: %v", ErrReflectorUnavailable, err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set(sentinelHeader, "1") + if r.bearer != "" { + req.Header.Set("Authorization", "Bearer "+r.bearer) + } + + resp, err := r.client.Do(req) + if err != nil { + return ReflectVerdict{}, fmt.Errorf("%w: %v", ErrReflectorUnavailable, err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return ReflectVerdict{}, fmt.Errorf("%w: status %d: %s", + ErrReflectorUnavailable, resp.StatusCode, preview(string(respBody), 240)) + } + + var v ReflectVerdict + if err := json.Unmarshal(respBody, &v); err != nil { + return ReflectVerdict{}, fmt.Errorf("%w: decode response: %v", ErrReflectorUnavailable, err) + } + if v.Decision == "" { + return ReflectVerdict{}, fmt.Errorf("%w: empty decision in response", ErrReflectorUnavailable) + } + return v, nil +} diff --git a/authbridge/authlib/plugins/sparc/reflector_test.go b/authbridge/authlib/plugins/sparc/reflector_test.go new file mode 100644 index 000000000..55a6c09db --- /dev/null +++ b/authbridge/authlib/plugins/sparc/reflector_test.go @@ -0,0 +1,92 @@ +package sparc + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestHTTPReflector_DecodesVerdict(t *testing.T) { + var gotPath, gotSentinel string + var gotBody ReflectInput + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotSentinel = r.Header.Get(sentinelHeader) + _ = json.NewDecoder(r.Body).Decode(&gotBody) + score := 2.0 + _ = json.NewEncoder(w).Encode(ReflectVerdict{ + Decision: DecisionReject, + OverallAvgScore: &score, + Issues: []ReflectIssue{ + {IssueType: "semantic_function", MetricName: "m", Explanation: "ungrounded id"}, + }, + }) + })) + defer srv.Close() + + r := newHTTPReflector(srv.URL, "", time.Second) + v, err := r.Reflect(context.Background(), ReflectInput{ + ToolCalls: []map[string]any{{"function": map[string]any{"name": "get_transaction"}}}, + SessionID: "s1", + }) + if err != nil { + t.Fatalf("Reflect: %v", err) + } + if gotPath != "/reflect" { + t.Errorf("path = %q, want /reflect", gotPath) + } + if gotSentinel != "1" { + t.Errorf("sentinel header = %q, want 1", gotSentinel) + } + if gotBody.SessionID != "s1" { + t.Errorf("forwarded session_id = %q, want s1", gotBody.SessionID) + } + if v.Decision != DecisionReject { + t.Errorf("decision = %q, want reject", v.Decision) + } + if v.OverallAvgScore == nil || *v.OverallAvgScore != 2.0 { + t.Errorf("score = %v, want 2.0", v.OverallAvgScore) + } + if len(v.Issues) != 1 || v.Issues[0].Explanation != "ungrounded id" { + t.Errorf("issues = %+v", v.Issues) + } +} + +func TestHTTPReflector_Non2xxIsUnavailable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, `{"error":"boom"}`, http.StatusBadGateway) + })) + defer srv.Close() + + r := newHTTPReflector(srv.URL, "", time.Second) + _, err := r.Reflect(context.Background(), ReflectInput{ToolCalls: []map[string]any{{}}}) + if !errors.Is(err, ErrReflectorUnavailable) { + t.Fatalf("expected ErrReflectorUnavailable, got %v", err) + } +} + +func TestHTTPReflector_EmptyDecisionIsUnavailable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"issues":[]}`)) + })) + defer srv.Close() + + r := newHTTPReflector(srv.URL, "", time.Second) + _, err := r.Reflect(context.Background(), ReflectInput{ToolCalls: []map[string]any{{}}}) + if !errors.Is(err, ErrReflectorUnavailable) { + t.Fatalf("expected ErrReflectorUnavailable for empty decision, got %v", err) + } +} + +func TestHTTPReflector_TransportErrorIsUnavailable(t *testing.T) { + // Point at a closed port to force a dial failure. + r := newHTTPReflector("http://127.0.0.1:1", "", 200*time.Millisecond) + _, err := r.Reflect(context.Background(), ReflectInput{ToolCalls: []map[string]any{{}}}) + if !errors.Is(err, ErrReflectorUnavailable) { + t.Fatalf("expected ErrReflectorUnavailable, got %v", err) + } +} diff --git a/authbridge/authlib/plugins/sparc/respond.go b/authbridge/authlib/plugins/sparc/respond.go new file mode 100644 index 000000000..7cfaaca5d --- /dev/null +++ b/authbridge/authlib/plugins/sparc/respond.go @@ -0,0 +1,123 @@ +package sparc + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/kagenti/kagenti-extensions/authbridge/authlib/pipeline" +) + +// buildClarificationText renders an agent-facing message from SPARC's issues. +// The agent receives this (as a tool result or rewritten completion) and relays +// it — typically asking the user for the missing / corrected information — +// without learning that SPARC intervened. +func buildClarificationText(verdict ReflectVerdict) string { + var b strings.Builder + b.WriteString("This tool call was not executed because it could not be verified against the conversation. ") + if len(verdict.Issues) == 0 { + b.WriteString("Please confirm the request details with the user before retrying.") + return b.String() + } + b.WriteString("Reason(s):") + for _, iss := range verdict.Issues { + if iss.Explanation == "" { + continue + } + fmt.Fprintf(&b, "\n- %s", iss.Explanation) + if c := correctionHint(iss.Correction); c != "" { + fmt.Fprintf(&b, " (suggested fix: %s)", c) + } + } + b.WriteString("\nAsk the user for the exact, missing, or corrected information, then retry.") + return b.String() +} + +func correctionHint(correction any) string { + if correction == nil { + return "" + } + if s, ok := correction.(string); ok { + return s + } + b, err := json.Marshal(correction) + if err != nil { + return "" + } + return preview(string(b), 200) +} + +// --- mcp mode: synthetic JSON-RPC MCP tool result --- + +// buildMCPResultBody crafts a JSON-RPC 2.0 MCP tools/call result envelope that +// carries the clarification as tool output. Returned with HTTP 200, the agent's +// MCP client consumes it as a normal tool result. +func buildMCPResultBody(rpcID any, text string) ([]byte, error) { + if rpcID == nil { + rpcID = 0 + } + return json.Marshal(map[string]any{ + "jsonrpc": "2.0", + "id": rpcID, + "result": map[string]any{ + "content": []map[string]any{{"type": "text", "text": text}}, + "isError": false, + "_meta": map[string]any{"sparc": map[string]any{"reflected": true}}, + }, + }) +} + +// mcpResultAction returns a Reject whose Violation renders as an HTTP 200 MCP +// result body. Reject is the only pipeline primitive that short-circuits the +// outbound call and synthesizes a response; the 200 + JSON-RPC result body turn +// that into a normal tool result. +func mcpResultAction(rpcID any, text string) pipeline.Action { + body, err := buildMCPResultBody(rpcID, text) + if err != nil { + return pipeline.DenyStatus(403, "sparc.blocked", text) + } + return pipeline.Action{ + Type: pipeline.Reject, + Violation: &pipeline.Violation{ + Code: "sparc.reflected", + Reason: "tool call replaced with a SPARC clarification result", + Status: 200, + Body: body, + BodyType: "application/json", + }, + } +} + +// --- inference mode: rewrite the OpenAI chat completion --- + +// rewriteInferenceResponse replaces the model's tool-call turn with a plain +// assistant message carrying `text`, in OpenAI chat-completion shape, via +// SetResponseBody. The agent receives a normal completion (no tool_call) and +// relays the clarification to the user. Preserves id/model/object from the +// upstream response when present. +func rewriteInferenceResponse(pctx *pipeline.Context, text string) { + assistant := map[string]any{"role": "assistant", "content": text} + + var orig map[string]any + if err := json.Unmarshal(pctx.ResponseBody, &orig); err == nil { + if choices, ok := orig["choices"].([]any); ok && len(choices) > 0 { + if c0, ok := choices[0].(map[string]any); ok { + c0["message"] = assistant + c0["finish_reason"] = "stop" + delete(c0, "delta") + choices[0] = c0 + orig["choices"] = choices[:1] + if out, err := json.Marshal(orig); err == nil { + pctx.SetResponseBody(out) + return + } + } + } + } + // Fallback: synthesize a minimal OpenAI completion. + out, _ := json.Marshal(map[string]any{ + "object": "chat.completion", + "choices": []map[string]any{{"index": 0, "message": assistant, "finish_reason": "stop"}}, + }) + pctx.SetResponseBody(out) +} diff --git a/authbridge/cmd/authbridge-envoy/main.go b/authbridge/cmd/authbridge-envoy/main.go index 7b995675c..8ebed98ee 100644 --- a/authbridge/cmd/authbridge-envoy/main.go +++ b/authbridge/cmd/authbridge-envoy/main.go @@ -53,6 +53,7 @@ import ( _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/inferenceparser" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/jwtvalidation" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/mcpparser" + _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/sparc" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/tokenbroker" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/tokenexchange" ) diff --git a/authbridge/cmd/authbridge-proxy/main.go b/authbridge/cmd/authbridge-proxy/main.go index 21d7b6865..5a4614b47 100644 --- a/authbridge/cmd/authbridge-proxy/main.go +++ b/authbridge/cmd/authbridge-proxy/main.go @@ -46,6 +46,7 @@ import ( _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/inferenceparser" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/jwtvalidation" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/mcpparser" + _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/sparc" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/tokenbroker" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/tokenexchange" ) diff --git a/authbridge/demos/README.md b/authbridge/demos/README.md index 9a8900332..389873436 100644 --- a/authbridge/demos/README.md +++ b/authbridge/demos/README.md @@ -22,6 +22,7 @@ more AuthBridge capabilities. | **[MCP Parser Plugin](mcp-parser/README.md)** | Reference | Enable the `mcp-parser` plugin to surface tool calls / resource reads in session events | Configuration only | | **[abctl Walkthrough](weather-agent/demo-with-abctl.md)** | Reference | Watch the AuthBridge plugin pipeline live with the `abctl` TUI | Tooling only | | **[IBAC](ibac/README.md)** | Intermediate | Intent-Based Access Control: LLM judge denies outbound HTTP that doesn't align with the user's recorded intent. Reproduces the email-poison / prompt-injection attack from `huang195/ibac`; chat with the agent through the kagenti UI and see the exfiltration blocked, then `make show-result` for a pipeline-level forensic | UI + kubectl | +| **[SPARC (finance)](finance-sparc/README.md)** | Intermediate | SPARC pre-tool reflection: the `sparc` plugin blocks a hallucinated/ungrounded tool argument (an invented transaction id) before it executes and transparently asks the user to clarify, then approves the corrected call. Complements IBAC — SPARC verifies argument grounding, IBAC verifies intent alignment | UI + kubectl | ## Recommended Path diff --git a/authbridge/demos/finance-sparc/.gitignore b/authbridge/demos/finance-sparc/.gitignore new file mode 100644 index 000000000..ab6901c56 --- /dev/null +++ b/authbridge/demos/finance-sparc/.gitignore @@ -0,0 +1 @@ +.stamps/ diff --git a/authbridge/demos/finance-sparc/Makefile b/authbridge/demos/finance-sparc/Makefile new file mode 100644 index 000000000..fe645499f --- /dev/null +++ b/authbridge/demos/finance-sparc/Makefile @@ -0,0 +1,213 @@ +# finance-sparc demo — SPARC pre-tool reflection on a regular finance agent. +# +# make demo # full end-to-end, watsonx (needs WX_* env) +# make demo PROVIDER=ollama # full end-to-end, local ollama (no creds) +# make undeploy # remove demo resources +# +# `make demo` is one-shot: it sets up the host (ollama reachable from the +# cluster), builds + loads images (incl. an authbridge image carrying the sparc +# plugin), deploys the SPARC service + finance MCP server + agent, enables the +# pipeline, configures Keycloak, and drives the two-turn scenario through REAL +# inbound auth (no jwt bypass), printing SPARC's verdicts. +# +# Prereqs: a kagenti kind cluster, Ollama on the host with the model, and +# kubectl / kind / docker(or podman) / python3 / uv. For PROVIDER=watsonx also +# export WX_API_KEY and WX_PROJECT_ID (e.g. `set -a; . /path/.env; set +a`). + +.PHONY: help preflight host-setup build-images build-authbridge load-images \ + ollama-up sparc-config deploy wait-pods patch-config setup-keycloak drive demo \ + build-abctl show-result undeploy logs-agent logs-sparc logs-ollama + +.DEFAULT_GOAL := help + +PROVIDER ?= watsonx +KIND_CLUSTER_NAME ?= kagenti +KIND_NODE := $(KIND_CLUSTER_NAME)-control-plane +# NS / AGENT / PLATFORM_NS are fixed: the k8s/ manifests hardcode these names +# (team1 / finance-agent / kagenti-system), so they are not override knobs — +# changing them here alone would target resources the manifests never create. +NS := team1 +AGENT := finance-agent +PLATFORM_NS := kagenti-system +# Prefer whichever runtime's daemon is actually reachable (docker first). +CONTAINER_RUNTIME ?= $(shell docker info >/dev/null 2>&1 && echo docker || (podman info >/dev/null 2>&1 && echo podman) || echo docker) +AUTHBRIDGE_IMAGE ?= ghcr.io/kagenti/kagenti-extensions/authbridge:v0.6.0-alpha.4 + +AGENT_IMAGE := finance-agent:latest +MCP_IMAGE := finance-mcp:latest +SPARC_IMAGE := sparc-service:latest +# Pinned (must match k8s/ollama.yaml). Override only if you stage a different tag. +OLLAMA_IMAGE := ollama/ollama:0.30.2 +# In-cluster Ollama (the agent's reasoning LLM for the watsonx path). +OLLAMA_URL := http://ollama.$(NS).svc.cluster.local:11434 + +# Host Ollama, reachable from the cluster. ALTK's ollama reflection is free-form +# generate+validate (guided decoding is disabled for ollama), which is slow on a +# CPU-only in-cluster node but fast on a Metal/GPU host. So PROVIDER=ollama uses +# the HOST Ollama for both reasoning and reflection. host.docker.internal is the +# portable default; on Rancher Desktop/colima use http://192.168.5.2:11434. +HOST_OLLAMA_URL ?= http://host.docker.internal:11434 + +# Per-provider models/timeouts (all override-able). +# - watsonx: reflection on watsonx (fast); reasoning on in-cluster Ollama (1b). +# - ollama: reflection AND reasoning on the host Ollama. Reflection needs a +# capable instruct model (3b/1b hallucinate the function-selection +# metric); reasoning needs a clean tool-caller. Override SPARC_MODEL/ +# AGENT_OLLAMA_MODEL to match what your host Ollama has pulled. +ifeq ($(PROVIDER),watsonx) +SPARC_MODEL ?= mistral-large-2512 +SPARC_TIMEOUT_MS ?= 30000 +TURN_TIMEOUT ?= 240 +AGENT_OLLAMA_URL ?= $(OLLAMA_URL) +AGENT_OLLAMA_MODEL ?= llama3.2:1b +else +SPARC_OLLAMA_URL ?= $(HOST_OLLAMA_URL) +SPARC_MODEL ?= gemma4:e4b +SPARC_TIMEOUT_MS ?= 200000 +TURN_TIMEOUT ?= 600 +AGENT_OLLAMA_URL ?= $(HOST_OLLAMA_URL) +AGENT_OLLAMA_MODEL ?= llama3.2:3b +endif + +help: ## Show this menu + @printf "\nfinance-sparc demo — SPARC reflection on a regular finance agent.\n\n" + @printf "Run:\n \033[1mmake demo\033[0m (watsonx; needs WX_* env)\n" + @printf " \033[1mmake demo PROVIDER=ollama\033[0m (local ollama; no creds)\n\n" + @printf "Targets:\n" + @awk 'BEGIN{FS=":.*## "} /^[a-zA-Z_-]+:.*## /{printf " \033[36m%-16s\033[0m %s\n",$$1,$$2}' $(MAKEFILE_LIST) + @printf "\nVars: PROVIDER(=%s) KIND_CLUSTER_NAME AUTHBRIDGE_IMAGE SPARC_MODEL HOST_OLLAMA_URL\n\n" "$(PROVIDER)" + +preflight: ## Verify kagenti + tools (+ watsonx creds when PROVIDER=watsonx) + @kubectl get ns $(PLATFORM_NS) >/dev/null 2>&1 || { echo "ERROR: '$(PLATFORM_NS)' not found — install kagenti first."; exit 1; } + @kubectl get ns $(NS) >/dev/null 2>&1 || { echo "ERROR: agent namespace '$(NS)' not found."; exit 1; } + @for t in kubectl kind python3 uv; do command -v $$t >/dev/null 2>&1 || { echo "ERROR: $$t not installed"; exit 1; }; done + @if [ "$(PROVIDER)" = "watsonx" ]; then \ + [ -n "$${WX_API_KEY:-}" ] && [ -n "$${WX_PROJECT_ID:-}" ] || { echo "ERROR: PROVIDER=watsonx needs WX_API_KEY/WX_PROJECT_ID (or run PROVIDER=ollama)"; exit 1; }; \ + fi + +host-setup: ## Ensure cluster internet egress works (MSS-clamp on hotspot/VPN networks) + KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) bash scripts/host-setup.sh + +ifeq ($(PROVIDER),watsonx) +ollama-up: ## watsonx: deploy in-cluster Ollama with the reasoning model (staged from host) + @if $(CONTAINER_RUNTIME) exec $(KIND_NODE) ctr -n k8s.io images list -q 2>/dev/null | grep -q "$(OLLAMA_IMAGE)"; then \ + echo "[*] $(OLLAMA_IMAGE) already present in node"; \ + else \ + $(CONTAINER_RUNTIME) image inspect $(OLLAMA_IMAGE) >/dev/null 2>&1 || $(CONTAINER_RUNTIME) pull $(OLLAMA_IMAGE); \ + kind load docker-image $(OLLAMA_IMAGE) --name $(KIND_CLUSTER_NAME); \ + $(CONTAINER_RUNTIME) exec $(KIND_NODE) ctr -n k8s.io images tag localhost/$(OLLAMA_IMAGE) docker.io/library/$(OLLAMA_IMAGE) >/dev/null 2>&1 || true; \ + fi + OLLAMA_MODEL=$(AGENT_OLLAMA_MODEL) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) bash scripts/stage-ollama-model.sh $(AGENT_OLLAMA_MODEL) + kubectl apply -f k8s/ollama.yaml + kubectl -n $(NS) rollout status deploy/ollama --timeout=180s + @echo "[*] warming $(AGENT_OLLAMA_MODEL) (loads into memory; KEEP_ALIVE keeps it resident) ..." + -kubectl -n $(NS) exec deploy/ollama -- ollama run $(AGENT_OLLAMA_MODEL) "hi" >/dev/null 2>&1 +else +ollama-up: ## ollama: use the host Ollama (Metal/GPU) for reasoning + reflection — no in-cluster Ollama + @echo "[*] PROVIDER=ollama: reasoning + reflection use the HOST Ollama at $(HOST_OLLAMA_URL)" + @echo " reasoning model=$(AGENT_OLLAMA_MODEL) reflection model=$(SPARC_MODEL)" + @echo " (ALTK reflects free-form on ollama; a CPU-only in-cluster Ollama is too slow, so we use the host.)" + @echo "[*] checking the cluster can reach the host Ollama ..." + @kubectl run sparc-ollama-probe-$$$$ --rm -i --restart=Never --image=curlimages/curl:8.10.1 --quiet -- \ + -s -m 10 -o /dev/null -w 'host Ollama %{http_code}\n' $(HOST_OLLAMA_URL)/api/tags 2>/dev/null \ + || echo " WARN: could not reach $(HOST_OLLAMA_URL) from the cluster. Ensure host Ollama runs (OLLAMA_HOST=0.0.0.0) and HOST_OLLAMA_URL is correct (Rancher/colima: http://192.168.5.2:11434)." +endif + +build-images: ## Build the agent, MCP server, and SPARC service images + @echo "[*] building $(SPARC_IMAGE)"; $(CONTAINER_RUNTIME) build -t $(SPARC_IMAGE) ../../sparc-service + @echo "[*] building $(AGENT_IMAGE)"; $(CONTAINER_RUNTIME) build -f finance-agent/Dockerfile -t $(AGENT_IMAGE) . + @echo "[*] building $(MCP_IMAGE)"; $(CONTAINER_RUNTIME) build -f finance-mcp/Dockerfile -t $(MCP_IMAGE) . + +build-authbridge: ## Build the authbridge image WITH the sparc plugin (operator's tag) + @echo "[*] building $(AUTHBRIDGE_IMAGE) (includes sparc plugin)" + cd ../.. && $(CONTAINER_RUNTIME) build -f cmd/authbridge-proxy/Dockerfile -t $(AUTHBRIDGE_IMAGE) . + +load-images: build-images build-authbridge ## Load all images into kind + @for img in $(AGENT_IMAGE) $(MCP_IMAGE) $(SPARC_IMAGE) $(AUTHBRIDGE_IMAGE); do \ + echo "[*] kind load $$img"; kind load docker-image "$$img" --name $(KIND_CLUSTER_NAME); \ + $(CONTAINER_RUNTIME) exec $(KIND_NODE) ctr -n k8s.io images tag localhost/$$img docker.io/library/$$img >/dev/null 2>&1 || true; \ + done + +sparc-config: ## Create the SPARC service ConfigMap (+ watsonx secret) for $(PROVIDER) + @if [ "$(PROVIDER)" = "watsonx" ]; then \ + kubectl -n $(PLATFORM_NS) create configmap sparc-service-config \ + --from-literal=SPARC_LLM_PROVIDER=watsonx --from-literal=SPARC_MODEL=$(SPARC_MODEL) \ + --from-literal=SPARC_TRACK=fast_track --from-literal=PORT=8090 \ + --dry-run=client -o yaml | kubectl apply -f - ; \ + kubectl -n $(PLATFORM_NS) create secret generic sparc-watsonx \ + --from-literal=WX_API_KEY="$${WX_API_KEY}" --from-literal=WX_PROJECT_ID="$${WX_PROJECT_ID}" \ + --from-literal=WX_URL="$${WX_URL:-https://us-south.ml.cloud.ibm.com}" \ + --dry-run=client -o yaml | kubectl apply -f - ; \ + else \ + kubectl -n $(PLATFORM_NS) create configmap sparc-service-config \ + --from-literal=SPARC_LLM_PROVIDER=ollama --from-literal=SPARC_MODEL=$(SPARC_MODEL) \ + --from-literal=SPARC_TRACK=fast_track --from-literal=PORT=8090 \ + --from-literal=OLLAMA_BASE_URL=$(SPARC_OLLAMA_URL) \ + --dry-run=client -o yaml | kubectl apply -f - ; \ + kubectl -n $(PLATFORM_NS) create secret generic sparc-watsonx --dry-run=client -o yaml | kubectl apply -f - ; \ + fi + @kubectl -n $(PLATFORM_NS) rollout restart deploy/sparc-service >/dev/null 2>&1 || true + +deploy: sparc-config ## Deploy the SPARC service, finance MCP server, and agent + kubectl apply -f k8s/sparc-service.yaml + kubectl apply -f k8s/finance-mcp.yaml + kubectl apply -f k8s/agent.yaml + @# Point the agent's reasoning LLM at the provider's Ollama (in-cluster for + @# watsonx, host for ollama). This rolls the agent; the operator then resets + @# its authbridge ConfigMap, which patch-config re-applies next. + kubectl -n $(NS) set env deploy/$(AGENT) -c agent OLLAMA_URL=$(AGENT_OLLAMA_URL) OLLAMA_MODEL=$(AGENT_OLLAMA_MODEL) >/dev/null + @# Pick up freshly-loaded images (apply is a no-op when only :latest content changed). + -kubectl -n $(NS) rollout restart deploy/finance-mcp >/dev/null 2>&1 + -kubectl -n $(NS) rollout restart deploy/$(AGENT) >/dev/null 2>&1 + $(MAKE) wait-pods + +wait-pods: ## Wait for service/MCP/agent + the operator-rendered authbridge ConfigMap + kubectl -n $(PLATFORM_NS) rollout status deploy/sparc-service --timeout=180s + kubectl -n $(NS) rollout status deploy/finance-mcp --timeout=120s + kubectl -n $(NS) rollout status deploy/$(AGENT) --timeout=180s + @for i in $$(seq 1 60); do kubectl -n $(NS) get cm authbridge-config-$(AGENT) >/dev/null 2>&1 && break; sleep 2; done + @kubectl -n $(NS) get cm authbridge-config-$(AGENT) >/dev/null 2>&1 || { echo "ERROR: operator didn't create authbridge-config-$(AGENT)"; exit 1; } + +patch-config: ## Enable the parsers + sparc in the agent's authbridge pipeline (hot-reload) + SPARC_TIMEOUT_MS=$(SPARC_TIMEOUT_MS) bash scripts/patch-sparc-config.sh $(NS) $(AGENT) + +setup-keycloak: ## Create the audience scope, ROPC client, and alice so inbound auth passes + @kubectl -n keycloak port-forward svc/keycloak-service 18081:8080 >/tmp/finance-kc-setup-pf.log 2>&1 & \ + KCPF=$$!; sleep 4; \ + NAMESPACE=$(NS) AGENT_SA=$(AGENT) KEYCLOAK_URL=http://localhost:18081 \ + uv run --with python-keycloak python scripts/setup_keycloak_finance.py; rc=$$?; \ + kill $$KCPF 2>/dev/null || true; exit $$rc + +drive: ## Drive the two-turn scenario through real auth and print SPARC verdicts + TURN_TIMEOUT=$(TURN_TIMEOUT) bash scripts/drive-demo.sh $(NS) $(AGENT) + +demo: preflight host-setup load-images ollama-up deploy patch-config setup-keycloak drive ## Full end-to-end demo + @echo + @echo "Demo complete. UI: http://kagenti-ui.localtest.me:8080 | forensic: make show-result" + +build-abctl: ## Build abctl into /tmp (for show-result) + cd ../../cmd/abctl && go build -o /tmp/abctl-finance-sparc-demo . + +show-result: build-abctl ## Launch abctl against the agent's session API + bash scripts/launch-abctl.sh $(NS) $(AGENT) + +undeploy: ## Remove demo resources (kagenti install untouched) + -kubectl -n $(NS) delete -f k8s/agent.yaml --ignore-not-found + -kubectl -n $(NS) delete -f k8s/finance-mcp.yaml --ignore-not-found + -kubectl -n $(NS) delete -f k8s/ollama.yaml --ignore-not-found + -kubectl delete -f k8s/sparc-service.yaml --ignore-not-found + -kubectl -n $(PLATFORM_NS) delete configmap sparc-service-config --ignore-not-found + -kubectl -n $(PLATFORM_NS) delete secret sparc-watsonx --ignore-not-found + -kubectl -n $(NS) delete configmap authbridge-config-$(AGENT) --ignore-not-found + -kubectl -n $(NS) delete secret -l kagenti.io/client-name=$(AGENT) --ignore-not-found + @echo "[*] demo resources removed." + +logs-agent: ## Tail agent + authbridge sidecar logs + @echo "--- agent ---"; kubectl -n $(NS) logs deploy/$(AGENT) -c agent --tail=80 + @echo "--- authbridge ---"; kubectl -n $(NS) logs deploy/$(AGENT) -c authbridge-proxy --tail=80 + +logs-sparc: ## Tail the SPARC reflection service logs + kubectl -n $(PLATFORM_NS) logs deploy/sparc-service --tail=80 + +logs-ollama: ## Tail the in-cluster Ollama logs + kubectl -n $(NS) logs deploy/ollama --tail=80 diff --git a/authbridge/demos/finance-sparc/README.md b/authbridge/demos/finance-sparc/README.md new file mode 100644 index 000000000..c2e20a558 --- /dev/null +++ b/authbridge/demos/finance-sparc/README.md @@ -0,0 +1,183 @@ +# finance-sparc demo — SPARC pre-tool reflection on a regular agent + +This demo shows the AuthBridge [`sparc` plugin](../../docs/sparc-plugin.md) catching a +hallucinated tool argument before it executes. SPARC's clarification is returned to the agent +as an MCP tool result, so the agent asks the user for the missing detail and then runs the +corrected call. + +The agent is an ordinary kagenti agent: it discovers its tools via MCP `tools/list`, reasons +with an Ollama model, and acts through an MCP finance backend. SPARC only sees what the agent +itself produces — the full conversation (including the system prompt), the discovered tool +specs in OpenAI format, and the proposed tool call. Those are collected by the parsers from the +agent's own LLM call; nothing is hardcoded or passed to SPARC out of band. + +## The scenario (two turns) + +1. "Refund my duplicate $450 subscription charge from last week." + The user gives no transaction id and there is no search tool, so the model invents a value + (e.g. `"$450"`) for the required `transaction_id` and calls `issue_refund`. SPARC + (`fast_track`) finds the argument ungrounded and rejects it; the plugin returns a + clarification as the tool result, and the agent relays it and asks for the exact id. +2. "The transaction id is TX4827. Please proceed." + Now the id is grounded in the conversation, SPARC approves, and the refund is issued. + +Example run (watsonx reflection): + +``` +Turn 1 agent> This tool call was not executed because it could not be verified... ask for the exact transaction_id +Turn 2 agent> Your transaction with ID TX4827 has been successfully refunded... + +SPARC modify/reflected tool=issue_refund score=0.00 # Turn 1: ungrounded "$450" → reject + clarify +SPARC allow/grounded tool=issue_refund score=1.00 # Turn 2: grounded TX4827 → approve → refund +``` + +## Prerequisites + +- A **kagenti kind cluster** with the AuthBridge operator. Core + SPIRE is enough: + ```bash + # from the kagenti repo: + scripts/kind/setup-kagenti.sh --with-spire + ``` +- `kubectl`, `kind`, `docker` (or `podman`), `python3`, `uv`. +- **Ollama models.** What you need depends on the reflection provider: + - **`make demo` (watsonx reflection):** just the agent's reasoning model in your host + Ollama store — `ollama pull llama3.2:1b`. The demo stages it into an *in-cluster* Ollama + (no host networking needed for this path). + - **`make demo PROVIDER=ollama` (local reflection):** a **running host Ollama** reachable + from the cluster, with a reasoning model and a **capable** reflection model: + ```bash + ollama pull llama3.2:3b # reasoning (clean tool calls) + ollama pull gemma4:e4b # reflection — needs a capable instruct model; 1b/3b + # hallucinate ALTK's function-selection metric. Override + # with SPARC_MODEL=. + ``` + Why the host Ollama for this path: ALTK reflects on ollama via **free-form + generate-then-validate** (it disables guided decoding for ollama), which is fast on a + Metal/GPU host but ~140s/call on a CPU-only in-cluster node. See [Provider notes](#provider-notes). + +## Run it + +```bash +cd authbridge/demos/finance-sparc + +# Option A — watsonx as the reflection LLM (best quality, fastest reflection): +set -a; . /path/to/your/.env; set +a # provides WX_API_KEY / WX_PROJECT_ID +make demo + +# Option B — fully local, no credentials (Ollama reflection on your host Ollama): +make demo PROVIDER=ollama +# Rancher Desktop / colima: the host Ollama isn't at host.docker.internal, so: +# make demo PROVIDER=ollama HOST_OLLAMA_URL=http://192.168.5.2:11434 +``` + +`make demo` is one-shot and idempotent. It: checks cluster egress (a no-op on healthy +networks), builds + loads images (including an authbridge image carrying the `sparc` plugin), +stages + warms the in-cluster Ollama model(s), deploys the SPARC service + finance MCP server + +agent, enables the pipeline, configures Keycloak, and drives the two-turn scenario **through +real inbound auth** (no jwt bypass), printing SPARC's verdicts. + +## How to read the output + +- **The two turns** print as `user>` / `agent>`. Turn 1's `agent>` is SPARC's clarification + (relayed by the agent); Turn 2's `agent>` is the successful refund. +- **The verdict summary** at the end (`SPARC / tool=… score=…`) is read straight + from the agent's AuthBridge session API: + - `modify/reflected` — SPARC rejected an ungrounded call and the clarification was returned. + - `allow/grounded` — SPARC approved a grounded call; the tool ran. + - `score` is SPARC's confidence (0=worst..1=best); shown only when the model returns one. +- **Full forensic view** — the per-plugin pipeline timeline and the structured SPARC event: + ```bash + make show-result # abctl TUI against the agent's session API + make logs-sparc # the SPARC reflection service logs + make logs-agent # agent + authbridge sidecar logs + ``` +- **Re-run just the conversation** (cluster already set up): `make drive`. +- **Tear down the demo** (kagenti install untouched): `make undeploy`. + +## How it works (and why it's generic) + +- The agent's LLM call goes through the AuthBridge forward proxy, where **`inference-parser`** + captures the messages (incl. the system prompt) and the OpenAI tool specs, and **`mcp-parser`** + captures the outbound `tools/call`. The **`sparc`** plugin correlates those into SPARC's three + inputs — no per-agent wiring. +- On a reject, the plugin returns SPARC's clarification as a JSON-RPC **MCP tool result** marked + `_meta.sparc.reflected`. The agent surfaces that to the user and stops re-trying the call + (honoring its system instruction to relay "could not verify / clarify" results) — exactly how + a well-behaved agent should treat a tool that asks for clarification. +- Reflection runs **out-of-band** (the plugin calls the SPARC service directly), so it is *not* + bound by the proxy's 30s upstream timeout. `make demo` sets a generous reflection `timeout_ms` + automatically for `PROVIDER=ollama`. + +## What's in here + +| Path | What | +|---|---| +| `finance-agent/` | A regular A2A agent: Ollama reasoning, MCP `tools/list` discovery, MCP tool execution. | +| `finance-mcp/` | MCP server: `get_transaction`, `lookup_customer`, `issue_refund`, `get_invoice`, `list_currencies` over a small multi-record dataset (real id: `TX4827`). | +| `k8s/sparc-patch.yaml` | Pipeline additions: `a2a-parser` inbound; `inference-parser` + `mcp-parser` + `sparc` outbound (mcp mode, fast_track, reflect, `skip_tools: [list_currencies]`). | +| `k8s/{agent,finance-mcp,sparc-service}.yaml` | Workload manifests. | +| `k8s/ollama.yaml` | In-cluster Ollama for the agent's reasoning model on the **watsonx** path; model staged onto the node — no host networking. (The `PROVIDER=ollama` path uses your host Ollama instead.) | +| `scripts/host-setup.sh` | Generic egress check (no-op on healthy networks; portable MSS clamp only if the link blackholes PMTUD). | +| `scripts/stage-ollama-model.sh` | Copies a model from the host Ollama store onto the kind node (local, no pull). | +| `scripts/setup_keycloak_finance.py` | Audience scope (realm default) + ROPC client + `alice` so inbound auth passes (scripted and UI). | +| `scripts/drive-demo.sh` | Gets a real user token and runs the two-turn scenario, printing SPARC verdicts. | +| `scripts/patch-sparc-config.sh` | Merges the pipeline patch into the operator's authbridge ConfigMap (hot-reload); substitutes the provider-tuned reflection timeout. | +| `Makefile` | `demo` (one-shot), `drive`, `show-result`, `undeploy`, `logs-*`. | + +## Provider notes + +- **watsonx (default)** — reflection on watsonx (`mistral-large-2512`, ~5s); the agent reasons + with an in-cluster Ollama (`llama3.2:1b`). Fastest, highest-quality, no host Ollama needed. + Verified: Turn 1 `modify/reflected score=0.00`, Turn 2 `allow/grounded score=1.00`. +- **ollama (`PROVIDER=ollama`)** — both reasoning and reflection run on your **host** Ollama. + Verified: Turn 1 `modify/reflected score=0.50`, Turn 2 `allow/grounded score=1.00`. + Two things make the host the right place for this path: + - **Reflection latency.** ALTK reflects on ollama via free-form generate-then-validate + (guided decoding is disabled for ollama in ALTK). That's ~140s/call on a CPU-only in-cluster + node (over the agent's 240s deadline → fails open), but a few seconds on a Metal/GPU host. + - **Reflection quality.** The function-selection metric needs a **capable** model. `llama3.2` + 1b/3b *hallucinate* it (they invent unrelated tools and wrongly reject grounded calls); + a larger instruct model (e.g. `gemma4:e4b`, the verified default) judges correctly. Override + with `SPARC_MODEL=`. + - The agent's reasoning model on this path is `llama3.2:3b` (a clean tool-caller; 1b tends to + emit malformed tool calls). Override with `AGENT_OLLAMA_MODEL=...`. + - `make demo PROVIDER=ollama` raises the reflection `timeout_ms` (200s) and per-turn timeout + (600s) to absorb first-call model loads. +- Small models are non-deterministic. The agent stops and relays on SPARC's clarification, so a + run won't loop; if the model ever declines to fabricate an id on Turn 1, re-run `make drive`. + +## Troubleshooting + +- **Image pulls fail with DNS errors** (`server misbehaving`, `could not resolve host`): your + container runtime's VM resolver went stale after a network change (common with Docker + Desktop / Rancher Desktop / colima after switching Wi-Fi or VPN). Restart the runtime, or + point its VM and the kind node at a public resolver (`8.8.8.8`). This is a host-runtime issue, + not a demo or kagenti one — the demo scripts contain no machine-specific networking. +- **`authbridge-config-` reverted to defaults**: the operator rewrites it whenever the + agent pod rolls. `make demo` (and `make patch-config`) re-apply the SPARC pipeline and wait + for the live config SHA to converge, so just re-run the relevant target. +- **Agent pods stuck `ContainerCreating` on a missing `kagenti-keycloak-client-credentials-*` + secret** (operator log: *"waiting for KEYCLOAK_URL/KEYCLOAK_REALM in authbridge-config"* or + *"waiting for keycloak-admin-secret"*): a platform/operator-version skew — the operator + registers the agent's Keycloak client from resources a matching chart version provides. This + affects **every** agent, not just this demo. Provide them once per cluster: + ```bash + kubectl -n team1 create configmap authbridge-config \ + --from-literal=KEYCLOAK_URL=http://keycloak-service.keycloak:8080 \ + --from-literal=KEYCLOAK_REALM=kagenti --dry-run=client -o yaml | kubectl apply -f - + kubectl -n kagenti-system create secret generic keycloak-admin-secret \ + --from-literal=KEYCLOAK_ADMIN_USERNAME=admin \ + --from-literal=KEYCLOAK_ADMIN_PASSWORD=admin --dry-run=client -o yaml | kubectl apply -f - + ``` + A kagenti install whose chart and operator versions are aligned creates these automatically. +- **`PROVIDER=ollama`: `reflector_unavailable` / can't reach the host Ollama**: the cluster + must reach your host Ollama. Run it with `OLLAMA_HOST=0.0.0.0`, and set `HOST_OLLAMA_URL` + correctly (Docker Desktop: `http://host.docker.internal:11434`; Rancher Desktop / colima: + `http://192.168.5.2:11434`). `make demo PROVIDER=ollama` probes reachability and warns if it + fails. A `reject` on the grounded turn usually means too small a reflection model — set + `SPARC_MODEL` to a capable instruct model your host Ollama has pulled. + +## See also +- [`docs/sparc-plugin.md`](../../docs/sparc-plugin.md) — plugin reference (config, modes). +- [`sparc-service/README.md`](../../sparc-service/README.md) — the reflection service. +- [`demos/ibac/`](../ibac/README.md) — complementary intent control (SPARC verifies grounding; IBAC verifies intent). diff --git a/authbridge/demos/finance-sparc/finance-agent/Dockerfile b/authbridge/demos/finance-sparc/finance-agent/Dockerfile new file mode 100644 index 000000000..f92c3e41c --- /dev/null +++ b/authbridge/demos/finance-sparc/finance-agent/Dockerfile @@ -0,0 +1,12 @@ +# Multi-stage build for the finance-sparc demo agent (stdlib only). +FROM golang:1.24-alpine AS build +WORKDIR /src +COPY go.mod ./ +COPY finance-agent/main.go ./finance-agent/main.go +RUN CGO_ENABLED=0 go build -o /out/agent ./finance-agent + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/agent /agent +# Listens on $PORT (default 8080). The demo Pod sets PORT=8000 to leave +# 8080 for the operator-injected authbridge sidecar's reverse proxy. +ENTRYPOINT ["/agent"] diff --git a/authbridge/demos/finance-sparc/finance-agent/main.go b/authbridge/demos/finance-sparc/finance-agent/main.go new file mode 100644 index 000000000..33c5cc3ed --- /dev/null +++ b/authbridge/demos/finance-sparc/finance-agent/main.go @@ -0,0 +1,471 @@ +// Finance assistant agent for the SPARC demo. +// +// A regular kagenti A2A agent (JSON-RPC over HTTP at /). It reasons with a +// local Ollama model and acts through a finance MCP server. It DISCOVERS its +// tools at runtime via MCP `tools/list` — nothing about the tools is hardcoded +// — so it behaves like any normal agent. There is no demo-specific steering or +// side-channel: SPARC sees only what the agent genuinely sends to its LLM (full +// conversation incl. system prompt, the discovered tool specs) and the tool +// call it makes. +// +// The agent is configured to be proactive (complete the user's request with the +// available tools). When a user refers to a charge descriptively — without a +// transaction id — the model fabricates a precise id to call a tool. The +// AuthBridge `sparc` plugin reflects on that call, finds the argument +// ungrounded, and returns a clarification as the MCP tool result; the agent +// relays it and asks the user for the exact id. Once supplied, SPARC approves +// and the refund proceeds. +package main + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" +) + +// --- OpenAI-compatible chat types (Ollama /v1/chat/completions) --- + +type ChatRequest struct { + Model string `json:"model"` + Messages []ChatMessage `json:"messages"` + Tools []Tool `json:"tools,omitempty"` + ToolChoice string `json:"tool_choice,omitempty"` + Temperature float64 `json:"temperature,omitempty"` +} + +type ChatMessage struct { + Role string `json:"role"` + Content string `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` +} + +type ToolCall struct { + ID string `json:"id"` + Type string `json:"type"` + Function FunctionCall `json:"function"` +} + +type FunctionCall struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +type Tool struct { + Type string `json:"type"` + Function ToolFunction `json:"function"` +} + +type ToolFunction struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters map[string]any `json:"parameters"` +} + +type ChatResponse struct { + Choices []ChatChoice `json:"choices"` +} + +type ChatChoice struct { + Message ChatMessage `json:"message"` +} + +// --- Outbound HTTP via the authbridge forward proxy --- + +var proxiedClient *http.Client + +func buildProxiedClient() *http.Client { + proxyEnv := os.Getenv("HTTP_PROXY") + if proxyEnv == "" { + log.Printf("[Agent] HTTP_PROXY unset — outbound HTTP will be direct (authbridge will not see it)") + return &http.Client{Timeout: 240 * time.Second} + } + u, err := url.Parse(proxyEnv) + if err != nil { + log.Printf("[Agent] HTTP_PROXY=%q invalid (%v) — direct", proxyEnv, err) + return &http.Client{Timeout: 240 * time.Second} + } + log.Printf("[Agent] outbound HTTP via proxy: %s", u) + return &http.Client{Timeout: 240 * time.Second, Transport: &http.Transport{Proxy: http.ProxyURL(u)}} +} + +// --- MCP (tool discovery + execution) --- + +func mcpURL() string { return envOr("FINANCE_MCP_URL", "http://localhost:8888") } + +func mcpCall(method string, params any) (map[string]any, error) { + rpc := map[string]any{"jsonrpc": "2.0", "id": fmt.Sprintf("%d", time.Now().UnixNano()), "method": method} + if params != nil { + rpc["params"] = params + } + body, _ := json.Marshal(rpc) + req, err := http.NewRequest(http.MethodPost, mcpURL()+"/mcp", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := proxiedClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + raw, _ := io.ReadAll(resp.Body) + var env map[string]any + if err := json.Unmarshal(raw, &env); err != nil { + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(raw)) + } + return env, nil +} + +var ( + toolsMu sync.Mutex + toolsCache []Tool +) + +// discoverTools fetches the MCP tool inventory via tools/list and maps it to +// OpenAI tool specs for the LLM. The result is cached only on success, so a +// transient failure (MCP not ready yet, a flaky call) is retried on the next +// turn rather than poisoning the agent with an empty tool list for its lifetime. +func discoverTools() []Tool { + toolsMu.Lock() + defer toolsMu.Unlock() + if len(toolsCache) > 0 { + return toolsCache + } + env, err := mcpCall("tools/list", nil) + if err != nil { + log.Printf("[Agent] tools/list failed (will retry): %v", err) + return nil + } + result, _ := env["result"].(map[string]any) + list, _ := result["tools"].([]any) + var discovered []Tool + for _, t := range list { + tm, _ := t.(map[string]any) + name, _ := tm["name"].(string) + if name == "" { + continue + } + desc, _ := tm["description"].(string) + params, _ := tm["inputSchema"].(map[string]any) + discovered = append(discovered, Tool{Type: "function", Function: ToolFunction{Name: name, Description: desc, Parameters: params}}) + } + if len(discovered) == 0 { + log.Printf("[Agent] tools/list returned no tools (will retry)") + return nil + } + toolsCache = discovered + log.Printf("[Agent] discovered %d tools via MCP tools/list", len(toolsCache)) + return toolsCache +} + +// execMCPTool runs a tool via tools/call and returns its text result (real data, +// or the SPARC clarification injected in place of an ungrounded call). The second +// return value is true when the result is a SPARC clarification (MCP `_meta.sparc. +// reflected`) — the agent surfaces that to the user rather than retrying the call. +func execMCPTool(name string, args map[string]any) (string, bool) { + env, err := mcpCall("tools/call", map[string]any{"name": name, "arguments": args}) + if err != nil { + return fmt.Sprintf("error calling tool: %v", err), false + } + if e, ok := env["error"].(map[string]any); ok { + return fmt.Sprintf("tool error: %v", e["message"]), false + } + result, _ := env["result"].(map[string]any) + reflected := false + if meta, ok := result["_meta"].(map[string]any); ok { + if sp, ok := meta["sparc"].(map[string]any); ok { + reflected, _ = sp["reflected"].(bool) + } + } + content, _ := result["content"].([]any) + var parts []string + for _, c := range content { + cm, _ := c.(map[string]any) + if txt, _ := cm["text"].(string); txt != "" { + parts = append(parts, txt) + } + } + if len(parts) == 0 { + return toJSONStr(result), reflected + } + return strings.Join(parts, "\n"), reflected +} + +func toJSONStr(v any) string { b, _ := json.Marshal(v); return string(b) } + +// --- Ollama interaction --- + +func callOllama(messages []ChatMessage, tools []Tool) (*ChatResponse, error) { + ollamaURL := envOr("OLLAMA_URL", "http://localhost:11434") + chatReq := ChatRequest{Model: envOr("OLLAMA_MODEL", "llama3.2:3b"), Messages: messages, Temperature: 0.1} + if len(tools) > 0 { + chatReq.Tools = tools + chatReq.ToolChoice = "auto" + } + reqBody, _ := json.Marshal(chatReq) + // Use the proxied client: it routes through the authbridge forward proxy + // (so inference-parser captures the messages + tool specs SPARC needs) and + // carries the explicit timeout — http.Post's default client has neither. + req, err := http.NewRequest(http.MethodPost, ollamaURL+"/v1/chat/completions", bytes.NewReader(reqBody)) + if err != nil { + return nil, fmt.Errorf("build ollama request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := proxiedClient.Do(req) + if err != nil { + return nil, fmt.Errorf("call ollama: %w", err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("ollama returned %d: %s", resp.StatusCode, string(body)) + } + var cr ChatResponse + if err := json.Unmarshal(body, &cr); err != nil { + return nil, fmt.Errorf("unmarshal ollama response: %w", err) + } + return &cr, nil +} + +func envOr(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +// --- Agent loop --- +// +// A normal proactive finance-assistant prompt. No id-guessing instructions and +// no ground-truth leakage — the hallucination, when it happens, comes from the +// model trying to act on a request that's missing a required argument. +const systemPrompt = "You are a finance operations assistant. Use the available tools to fulfil the user's " + + "request. To refund a charge you must call issue_refund with the transaction's id. Take action with the " + + "tools rather than replying in prose. If a tool result tells you a value could not be verified or asks you " + + "to clarify, relay that to the user and ask for the exact missing detail." + +func runAgent(query string) (string, error) { + tools := discoverTools() + messages := []ChatMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: query}, + } + lastToolResult := "" + for i := 0; i < 8; i++ { + resp, err := callOllama(messages, tools) + if err != nil { + return "", err + } + if len(resp.Choices) == 0 { + return "", fmt.Errorf("no choices in ollama response") + } + msg := resp.Choices[0].Message + if len(msg.ToolCalls) == 0 { + return msg.Content, nil + } + messages = append(messages, msg) + for _, tc := range msg.ToolCalls { + log.Printf("[Agent] tool call: %s(%s)", tc.Function.Name, tc.Function.Arguments) + var args map[string]any + if err := json.Unmarshal([]byte(tc.Function.Arguments), &args); err != nil { + args = map[string]any{} + } + result, reflected := execMCPTool(tc.Function.Name, args) + lastToolResult = result + log.Printf("[Agent] tool result (%s, reflected=%t): %.200s", tc.Function.Name, reflected, result) + messages = append(messages, ChatMessage{Role: "tool", Content: result, ToolCallID: tc.ID}) + if reflected { + // SPARC returned a clarification in place of the tool result + // (an ungrounded/unverifiable argument). Relay it to the user + // and stop — retrying would hit the same rejection and only + // grow the prompt. This is the agent honoring its system + // instruction to surface "could not verify / clarify" results. + return result, nil + } + } + } + // The model kept calling tools without concluding (common with small models + // when a tool keeps returning a clarification). Surface the last tool result + // to the user rather than a bare error — it carries the actionable guidance. + if lastToolResult != "" { + return lastToolResult, nil + } + return "", fmt.Errorf("tool-calling loop exceeded max iterations") +} + +// --- A2A (JSON-RPC 2.0) endpoint --- + +type a2aRequest struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id"` + Method string `json:"method"` + Params struct { + Message struct { + Role string `json:"role"` + Parts []part `json:"parts"` + ContextID string `json:"contextId,omitempty"` + } `json:"message"` + } `json:"params"` +} + +type part struct { + Kind string `json:"kind"` + Text string `json:"text,omitempty"` +} + +type a2aResponse struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id"` + Result *a2aTask `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +type a2aTask struct { + ID string `json:"id"` + ContextID string `json:"contextId,omitempty"` + Kind string `json:"kind"` + Status a2aStatus `json:"status"` + Artifacts []a2aArtifact `json:"artifacts,omitempty"` +} + +type a2aStatus struct { + State string `json:"state"` + Message *a2aMessage `json:"message,omitempty"` +} + +type a2aMessage struct { + Role string `json:"role"` + Parts []part `json:"parts"` +} + +type a2aArtifact struct { + ArtifactID string `json:"artifactId"` + Name string `json:"name,omitempty"` + Parts []part `json:"parts"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +func handleAgentCard(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + publicURL := envOr("AGENT_PUBLIC_URL", "http://finance-agent.team1.svc.cluster.local:8080/") + card := map[string]any{ + "name": "Finance Assistant", + "description": "Finance operations agent: transaction lookups and refunds via an MCP finance backend. AuthBridge's SPARC plugin reflects on each tool call and blocks ungrounded/hallucinated arguments, transparently asking for clarification.", + "protocolVersion": "0.3.0", + "version": "0.0.1", + "url": publicURL, + "preferredTransport": "JSONRPC", + "defaultInputModes": []string{"text"}, + "defaultOutputModes": []string{"text"}, + "capabilities": map[string]any{"streaming": false}, + "skills": []map[string]any{{ + "id": "refund", "name": "Refund a transaction", + "description": "Looks up and refunds a transaction; SPARC verifies the transaction id is grounded before any refund runs.", + "tags": []string{"demo", "finance", "sparc"}, + "examples": []string{"Refund my duplicate $450 subscription charge from last week."}, + }}, + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(card); err != nil { + log.Printf("[Agent] encode agent card: %v", err) + } +} + +func handleA2A(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "read body", http.StatusBadRequest) + return + } + var req a2aRequest + if err := json.Unmarshal(body, &req); err != nil { + writeRPCError(w, nil, -32700, "parse error: "+err.Error()) + return + } + if req.Method != "message/send" { + writeRPCError(w, req.ID, -32601, "method not found: "+req.Method) + return + } + var query string + for _, p := range req.Params.Message.Parts { + if p.Kind == "text" && p.Text != "" { + query = p.Text + break + } + } + if query == "" { + writeRPCError(w, req.ID, -32602, "no text part in message") + return + } + sessionID := req.Params.Message.ContextID + if sessionID == "" { + sessionID = newUUID() + } + log.Printf("[Agent] A2A query (session=%s): %s", sessionID, query) + result, err := runAgent(query) + if err != nil { + writeRPCError(w, req.ID, -32603, err.Error()) + return + } + writeRPCSuccess(w, req.ID, sessionID, result) +} + +func writeRPCSuccess(w http.ResponseWriter, id any, sessionID, text string) { + parts := []part{{Kind: "text", Text: text}} + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(a2aResponse{ + JSONRPC: "2.0", ID: id, + Result: &a2aTask{ + ID: newUUID(), ContextID: sessionID, Kind: "task", + Status: a2aStatus{State: "completed", Message: &a2aMessage{Role: "agent", Parts: parts}}, + Artifacts: []a2aArtifact{{ArtifactID: newUUID(), Name: "reply", Parts: parts}}, + }, + }) +} + +func newUUID() string { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return fmt.Sprintf("%d", time.Now().UnixNano()) + } + return hex.EncodeToString(b[:]) +} + +func writeRPCError(w http.ResponseWriter, id any, code int, message string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(a2aResponse{JSONRPC: "2.0", ID: id, Error: &rpcError{Code: code, Message: message}}) +} + +func main() { + proxiedClient = buildProxiedClient() + http.HandleFunc("/", handleA2A) + http.HandleFunc("/.well-known/agent-card.json", handleAgentCard) + port := envOr("PORT", "8080") + log.Printf("[Agent] Finance assistant starting on :%s (A2A at /)", port) + if err := http.ListenAndServe(":"+port, nil); err != nil { //nolint:gosec // demo + log.Fatalf("server: %v", err) + } +} diff --git a/authbridge/demos/finance-sparc/finance-mcp/Dockerfile b/authbridge/demos/finance-sparc/finance-mcp/Dockerfile new file mode 100644 index 000000000..1d0db94b8 --- /dev/null +++ b/authbridge/demos/finance-sparc/finance-mcp/Dockerfile @@ -0,0 +1,11 @@ +# Multi-stage build for the finance-sparc demo MCP server (stdlib only). +FROM golang:1.24-alpine AS build +WORKDIR /src +COPY go.mod ./ +COPY finance-mcp/main.go ./finance-mcp/main.go +RUN CGO_ENABLED=0 go build -o /out/finance-mcp ./finance-mcp + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/finance-mcp /finance-mcp +EXPOSE 8888 +ENTRYPOINT ["/finance-mcp"] diff --git a/authbridge/demos/finance-sparc/finance-mcp/main.go b/authbridge/demos/finance-sparc/finance-mcp/main.go new file mode 100644 index 000000000..c40104d91 --- /dev/null +++ b/authbridge/demos/finance-sparc/finance-mcp/main.go @@ -0,0 +1,220 @@ +// Finance MCP server for the SPARC demo. +// +// Speaks MCP (Model Context Protocol) over HTTP — JSON-RPC 2.0 POSTs to /mcp. +// Serves a small but non-trivial finance dataset through five tools and +// advertises them via tools/list (so the agent discovers them at runtime like +// any normal kagenti agent — nothing is hardcoded on the agent side). +// +// Tools: +// +// get_transaction(transaction_id) → transaction details +// lookup_customer(customer_id) → customer details +// issue_refund(transaction_id[, reason]) → refund confirmation (id required) +// get_invoice(invoice_id) → invoice details +// list_currencies() → static reference data (demo of skip_tools) +// +// There is deliberately NO search/list-transactions tool: when a user refers to +// a charge descriptively (no id), the agent cannot look the id up — a proactive +// model will fabricate a precise transaction_id, which SPARC catches as +// ungrounded before any refund touches the wrong record. +package main + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "strings" +) + +type jsonRPCRequest struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params,omitempty"` +} + +type jsonRPCResponse struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id"` + Result any `json:"result,omitempty"` + Error *jsonRPCError `json:"error,omitempty"` +} + +type jsonRPCError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type mcpContent struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` +} + +type mcpToolCallResult struct { + Content []mcpContent `json:"content"` + IsError bool `json:"isError,omitempty"` +} + +type mcpToolCallParams struct { + Name string `json:"name"` + Arguments map[string]any `json:"arguments,omitempty"` +} + +// --- In-memory finance dataset (several of each entity) --- + +var transactions = map[string]map[string]any{ + "TX4827": {"transaction_id": "TX4827", "amount": 450.00, "currency": "USD", "customer_id": "C921", "status": "settled", "description": "Annual subscription renewal"}, + "TX5310": {"transaction_id": "TX5310", "amount": 1200.00, "currency": "USD", "customer_id": "C742", "status": "settled", "description": "Hardware purchase"}, + "TX2981": {"transaction_id": "TX2981", "amount": 75.50, "currency": "EUR", "customer_id": "C921", "status": "pending", "description": "Bookstore order"}, + "TX6644": {"transaction_id": "TX6644", "amount": 99.00, "currency": "USD", "customer_id": "C305", "status": "settled", "description": "Monthly subscription"}, +} + +var customers = map[string]map[string]any{ + "C921": {"customer_id": "C921", "name": "Daniel Reed", "email": "daniel.reed@example.com", "tier": "gold"}, + "C742": {"customer_id": "C742", "name": "Maria Gomez", "email": "maria.gomez@example.com", "tier": "silver"}, + "C305": {"customer_id": "C305", "name": "Sven Olsen", "email": "sven.olsen@example.com", "tier": "bronze"}, +} + +var invoices = map[string]map[string]any{ + "INV-8834": {"invoice_id": "INV-8834", "vendor": "Acme Corp", "amount": 1280.00, "currency": "USD", "status": "due"}, + "INV-9102": {"invoice_id": "INV-9102", "vendor": "Globex", "amount": 540.00, "currency": "USD", "status": "paid"}, +} + +func toJSON(v any) string { b, _ := json.Marshal(v); return string(b) } + +func strProp(desc string) map[string]any { + return map[string]any{"type": "string", "description": desc} +} + +func fn(name, desc string, props map[string]any, required []string) map[string]any { + return map[string]any{ + "name": name, + "description": desc, + "parameters": map[string]any{"type": "object", "properties": props, "required": required}, + } +} + +// toolSpecs advertised via tools/list (OpenAI function-calling shape under +// inputSchema, the MCP convention the agent maps to its LLM tool list). +func toolSpecs() []map[string]any { + return []map[string]any{ + fn("get_transaction", "Fetch transaction details by exact transaction id.", + map[string]any{"transaction_id": strProp("Exact transaction identifier")}, []string{"transaction_id"}), + fn("lookup_customer", "Fetch customer details by exact customer id.", + map[string]any{"customer_id": strProp("Exact customer identifier")}, []string{"customer_id"}), + fn("issue_refund", "Issue a refund for a transaction. Requires the exact transaction_id.", + map[string]any{ + "transaction_id": strProp("Exact transaction identifier to refund"), + "reason": strProp("Optional reason for the refund"), + }, []string{"transaction_id"}), + fn("get_invoice", "Fetch an invoice by exact invoice id.", + map[string]any{"invoice_id": strProp("Exact invoice identifier")}, []string{"invoice_id"}), + fn("list_currencies", "List supported reference currencies (static data).", + map[string]any{}, []string{}), + } +} + +func callTool(name string, args map[string]any) mcpToolCallResult { + get := func(k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + notFound := func(kind, id string) mcpToolCallResult { + return mcpToolCallResult{Content: []mcpContent{{Type: "text", Text: fmt.Sprintf("%s %q not found.", kind, id)}}, IsError: true} + } + ok := func(v any) mcpToolCallResult { + return mcpToolCallResult{Content: []mcpContent{{Type: "text", Text: toJSON(v)}}} + } + switch name { + case "get_transaction": + if t, found := transactions[get("transaction_id")]; found { + return ok(t) + } + return notFound("transaction", get("transaction_id")) + case "lookup_customer": + if c, found := customers[get("customer_id")]; found { + return ok(c) + } + return notFound("customer", get("customer_id")) + case "issue_refund": + tid := get("transaction_id") + if _, found := transactions[tid]; !found { + return notFound("transaction", tid) + } + return ok(map[string]any{"status": "refund_issued", "transaction_id": tid, "reason": get("reason")}) + case "get_invoice": + if inv, found := invoices[get("invoice_id")]; found { + return ok(inv) + } + return notFound("invoice", get("invoice_id")) + case "list_currencies": + return ok([]string{"USD", "EUR", "GBP", "JPY"}) + default: + return mcpToolCallResult{Content: []mcpContent{{Type: "text", Text: "unknown tool: " + name}}, IsError: true} + } +} + +func writeError(w http.ResponseWriter, id any, code int, message string) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(jsonRPCResponse{JSONRPC: "2.0", ID: id, Error: &jsonRPCError{Code: code, Message: message}}) +} + +func writeResult(w http.ResponseWriter, id any, result any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(jsonRPCResponse{JSONRPC: "2.0", ID: id, Result: result}) +} + +func handleMCP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + writeError(w, nil, -32700, "read body: "+err.Error()) + return + } + var req jsonRPCRequest + if err := json.Unmarshal(body, &req); err != nil { + writeError(w, nil, -32700, "parse error: "+err.Error()) + return + } + slog.Info("mcp request", "method", req.Method, "id", req.ID) + switch req.Method { + case "initialize": + writeResult(w, req.ID, map[string]any{ + "protocolVersion": "2024-11-05", + "capabilities": map[string]any{"tools": map[string]any{}}, + "serverInfo": map[string]any{"name": "finance-mcp", "version": "0.1.0"}, + }) + case "tools/list": + tools := make([]map[string]any, 0) + for _, s := range toolSpecs() { + tools = append(tools, map[string]any{"name": s["name"], "description": s["description"], "inputSchema": s["parameters"]}) + } + writeResult(w, req.ID, map[string]any{"tools": tools}) + case "tools/call": + var params mcpToolCallParams + if len(req.Params) > 0 { + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + writeError(w, req.ID, -32602, "params: "+err.Error()) + return + } + } + res := callTool(params.Name, params.Arguments) + slog.Info("tools/call", "tool", params.Name, "args", toJSON(params.Arguments), "isError", res.IsError) + writeResult(w, req.ID, res) + default: + writeError(w, req.ID, -32601, "method not found: "+req.Method) + } +} + +func main() { + http.HandleFunc("/mcp", handleMCP) + addr := ":8888" + slog.Info("finance MCP server starting", "addr", addr+"/mcp") + if err := http.ListenAndServe(addr, nil); err != nil { //nolint:gosec // demo + slog.Error("failed to start finance server", "error", err) + os.Exit(1) + } +} diff --git a/authbridge/demos/finance-sparc/go.mod b/authbridge/demos/finance-sparc/go.mod new file mode 100644 index 000000000..d4bebc7cc --- /dev/null +++ b/authbridge/demos/finance-sparc/go.mod @@ -0,0 +1,3 @@ +module github.com/kagenti/kagenti-extensions/authbridge/demos/finance-sparc + +go 1.24 diff --git a/authbridge/demos/finance-sparc/k8s/agent.yaml b/authbridge/demos/finance-sparc/k8s/agent.yaml new file mode 100644 index 000000000..59f62b567 --- /dev/null +++ b/authbridge/demos/finance-sparc/k8s/agent.yaml @@ -0,0 +1,121 @@ +# finance-sparc demo agent — operator-injected, kagenti-UI-discoverable. +# +# Required labels (same contract as demos/ibac/k8s/agent.yaml): +# kagenti.io/type: agent — UI lists agents by this label +# kagenti.io/inject: enabled — operator's webhook injects authbridge +# kagenti.io/spire: disabled — demo doesn't depend on SPIRE identity +# protocol.kagenti.io/a2a: "" — declares A2A for the chat flow +# +# The operator port-steals (containerPort 8000 → agent on 8001, PORT and +# HTTP_PROXY set so outbound flows through the injected sidecar's forward +# proxy). We do NOT ship a per-agent authbridge ConfigMap — the operator +# creates authbridge-config-finance-agent; scripts/patch-sparc-config.sh +# enables the parsers + sparc plugin after deploy. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: finance-agent + namespace: team1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: finance-agent + namespace: team1 + labels: + app.kubernetes.io/name: finance-agent + app.kubernetes.io/part-of: finance-sparc-demo + kagenti.io/type: agent + protocol.kagenti.io/a2a: "" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: finance-agent + template: + metadata: + labels: + app.kubernetes.io/name: finance-agent + app.kubernetes.io/part-of: finance-sparc-demo + kagenti.io/type: agent + kagenti.io/inject: enabled + kagenti.io/spire: disabled + protocol.kagenti.io/a2a: "" + spec: + serviceAccountName: finance-agent + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: agent + image: finance-agent:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8000 + name: http + env: + - name: PORT + value: "8000" + - name: OLLAMA_URL + value: "http://ollama.team1.svc.cluster.local:11434" + - name: OLLAMA_MODEL + value: "llama3.2:1b" + - name: FINANCE_MCP_URL + value: "http://finance-mcp.team1.svc.cluster.local:8888" + - name: AGENT_PUBLIC_URL + value: "http://finance-agent.team1.svc.cluster.local:8080/" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 50m + memory: 64Mi + # /shared is mounted by the operator-injected authbridge sidecar + # (it reads /shared/client-id.txt). The agent doesn't read it; + # the volume just has to exist on the Pod. + volumeMounts: + - name: shared-data + mountPath: /shared + volumes: + - name: shared-data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: finance-agent + namespace: team1 + labels: + app.kubernetes.io/name: finance-agent + app.kubernetes.io/part-of: finance-sparc-demo +spec: + selector: + app.kubernetes.io/name: finance-agent + ports: + - port: 8080 + targetPort: 8000 + protocol: TCP + name: http + type: ClusterIP +--- +# AgentRuntime triggers operator-managed Keycloak client registration +# (the Secret the authbridge sidecar mounts at /shared/). The operator +# auto-creates the AgentCard CR for the kagenti.io/type=agent Deployment. +apiVersion: agent.kagenti.dev/v1alpha1 +kind: AgentRuntime +metadata: + name: finance-agent + namespace: team1 +spec: + type: agent + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: finance-agent diff --git a/authbridge/demos/finance-sparc/k8s/finance-mcp.yaml b/authbridge/demos/finance-sparc/k8s/finance-mcp.yaml new file mode 100644 index 000000000..208a4474a --- /dev/null +++ b/authbridge/demos/finance-sparc/k8s/finance-mcp.yaml @@ -0,0 +1,63 @@ +# Finance MCP server — the tool backend the finance-agent calls. +# Plain in-cluster workload (no authbridge injection); the agent reaches +# it at http://finance-mcp.team1.svc.cluster.local:8888/mcp. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: finance-mcp + namespace: team1 + labels: + app.kubernetes.io/name: finance-mcp + app.kubernetes.io/part-of: finance-sparc-demo +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: finance-mcp + template: + metadata: + labels: + app.kubernetes.io/name: finance-mcp + app.kubernetes.io/part-of: finance-sparc-demo + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: finance-mcp + image: finance-mcp:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8888 + name: http + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + limits: + cpu: 200m + memory: 64Mi + requests: + cpu: 20m + memory: 16Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: finance-mcp + namespace: team1 + labels: + app.kubernetes.io/name: finance-mcp + app.kubernetes.io/part-of: finance-sparc-demo +spec: + selector: + app.kubernetes.io/name: finance-mcp + ports: + - port: 8888 + targetPort: 8888 + protocol: TCP + name: http + type: ClusterIP diff --git a/authbridge/demos/finance-sparc/k8s/ollama.yaml b/authbridge/demos/finance-sparc/k8s/ollama.yaml new file mode 100644 index 000000000..dc1f4fecf --- /dev/null +++ b/authbridge/demos/finance-sparc/k8s/ollama.yaml @@ -0,0 +1,95 @@ +# In-cluster Ollama for the demo — provides the agent's reasoning LLM on the +# watsonx path WITHOUT depending on host networking. The model is staged onto +# the kind node at /opt/ollama-models (by scripts/stage-ollama-model.sh) and +# mounted here, so no large pull is needed. (The PROVIDER=ollama path uses the +# host Ollama instead and doesn't deploy this workload.) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ollama + namespace: team1 + labels: + app.kubernetes.io/name: ollama + app.kubernetes.io/part-of: finance-sparc-demo +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: ollama + template: + metadata: + labels: + app.kubernetes.io/name: ollama + app.kubernetes.io/part-of: finance-sparc-demo + kagenti.io/inject: disabled + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: ollama + image: ollama/ollama:0.30.2 + imagePullPolicy: IfNotPresent + env: + - name: OLLAMA_HOST + value: "0.0.0.0" + - name: OLLAMA_KEEP_ALIVE + value: "24h" + # Run as a non-root uid: keep HOME (keys/history) on a writable + # emptyDir and read the staged models from the mounted hostPath. + - name: HOME + value: /home/ollama + - name: OLLAMA_MODELS + value: /models + ports: + - containerPort: 11434 + name: http + volumeMounts: + - name: models + mountPath: /models + readOnly: true + - name: home + mountPath: /home/ollama + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readinessProbe: + httpGet: + path: /api/tags + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 12 + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + memory: 8Gi + volumes: + - name: models + hostPath: + path: /opt/ollama-models + type: Directory + - name: home + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: ollama + namespace: team1 + labels: + app.kubernetes.io/name: ollama + app.kubernetes.io/part-of: finance-sparc-demo +spec: + selector: + app.kubernetes.io/name: ollama + ports: + - port: 11434 + targetPort: http + name: http diff --git a/authbridge/demos/finance-sparc/k8s/sparc-patch.yaml b/authbridge/demos/finance-sparc/k8s/sparc-patch.yaml new file mode 100644 index 000000000..5a5e5927a --- /dev/null +++ b/authbridge/demos/finance-sparc/k8s/sparc-patch.yaml @@ -0,0 +1,53 @@ +# SPARC plugin-pipeline patch fragment. +# +# NOT a Kubernetes manifest. scripts/patch-sparc-config.sh merges these +# additions into the operator-rendered authbridge ConfigMap +# (authbridge-config-finance-agent in team1): +# - prepend a2a-parser to inbound (so the session carries the user turn) +# - append inference-parser + mcp-parser + sparc to outbound +# inference-parser: captures the agent's LLM call (messages incl. system +# prompt + tool specs) into the session — SPARC's generic +# conversation/tool-spec source +# mcp-parser: parses the outbound MCP tools/call (the tool call) +# sparc: reflects and enforces (mcp mode → MCP tool result) + +inbound_prepend: + - name: a2a-parser + +outbound_append: + - name: inference-parser + - name: mcp-parser + - name: sparc + config: + # The in-cluster SPARC reflection service. The plugin POSTs to {endpoint}/reflect. + # ${...} placeholders are substituted by scripts/patch-sparc-config.sh. + reflector_endpoint: "${SPARC_REFLECTOR_ENDPOINT}" + + # Gate the MCP tools/call and return the clarification as an MCP tool + # result (the kagenti norm). Use "inference" for non-MCP agents. + enforcement: "mcp" + + # fast_track = hallucination + function-selection checks. + track: "fast_track" + + # Return SPARC's reflection to the agent on a reject (reflection mode): + # observe = log only; reflect = return the clarification; deny = hard block. + on_reject_action: "reflect" + + # Escalate a reject to a hard deny when SPARC's score (1=worst..5=best) is + # this low. 0 disables (reflect handles all rejects). + deny_score_threshold: 0 + + # Don't reflect on trivial read-only tools — saves a round-trip. + skip_tools: + - "list_currencies" + + # SPARC is a quality gate, not auth: if the service is unreachable, allow + # and log rather than break the agent. + fail_policy: "open" + + # Max time the plugin waits for a reflection verdict. watsonx answers in + # ~5s; a CPU-only in-cluster Ollama is slower, so the Makefile raises this + # for PROVIDER=ollama. The plugin reflects out-of-band, so this is NOT + # bounded by the forward proxy's upstream timeout. + timeout_ms: ${SPARC_TIMEOUT_MS} diff --git a/authbridge/demos/finance-sparc/k8s/sparc-service.yaml b/authbridge/demos/finance-sparc/k8s/sparc-service.yaml new file mode 100644 index 000000000..a99bb6091 --- /dev/null +++ b/authbridge/demos/finance-sparc/k8s/sparc-service.yaml @@ -0,0 +1,82 @@ +# SPARC reflection service for the demo, deployed into kagenti-system. +# Its config (provider/model/track) is supplied by the `sparc-service-config` +# ConfigMap and the optional `sparc-watsonx` Secret, both created by the +# Makefile (provider-aware: watsonx or ollama). In production this ships via the +# kagenti Helm chart (components.sparcService.enabled=true). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sparc-service + namespace: kagenti-system + labels: + app.kubernetes.io/name: sparc-service + app.kubernetes.io/part-of: finance-sparc-demo +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: sparc-service + template: + metadata: + labels: + app.kubernetes.io/name: sparc-service + app.kubernetes.io/part-of: finance-sparc-demo + kagenti.io/inject: disabled + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sparc-service + image: sparc-service:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8090 + name: http + envFrom: + - configMapRef: + name: sparc-service-config + - secretRef: + name: sparc-watsonx + optional: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 12 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 5 + periodSeconds: 20 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: sparc-service + namespace: kagenti-system + labels: + app.kubernetes.io/name: sparc-service + app.kubernetes.io/part-of: finance-sparc-demo +spec: + selector: + app.kubernetes.io/name: sparc-service + ports: + - port: 8090 + targetPort: http + name: http diff --git a/authbridge/demos/finance-sparc/scripts/drive-demo.sh b/authbridge/demos/finance-sparc/scripts/drive-demo.sh new file mode 100644 index 000000000..7ae243d8b --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/drive-demo.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Drive the finance-sparc demo end-to-end through REAL inbound auth (no jwt +# bypass). Gets a user token from Keycloak (ROPC), then sends the two-turn +# refund scenario to the agent through its AuthBridge sidecar, and prints the +# SPARC verdicts recorded in the session. +# +# Usage: drive-demo.sh [namespace] [agent] +set -euo pipefail + +NS=${1:-team1} +AGENT=${2:-finance-agent} +REALM=${KEYCLOAK_REALM:-kagenti} +ROPC_CLIENT_ID=${ROPC_CLIENT_ID:-finance-sparc-e2e} +USER=${DEMO_USER:-alice} +PASS=${DEMO_PASS:-alice123} +CTX="finance-demo-$(date +%s)" +PORT=${AGENT_PORT:-18080} +KCPORT=${KC_PORT:-18081} + +say() { printf '\n\033[1m%s\033[0m\n' "$*"; } + +# Reach Keycloak via kubectl port-forward (portable; doesn't depend on the +# cluster ingress / host port-forward). Keycloak's fixed KC_HOSTNAME makes the +# token's issuer correct regardless of how it's reached. +say "[1/4] Obtaining a user token from Keycloak (via port-forward) ..." +kubectl -n keycloak port-forward svc/keycloak-service "$KCPORT:8080" >/tmp/finance-kc-pf.log 2>&1 & +KCPF=$!; trap 'kill $KCPF 2>/dev/null || true' EXIT +sleep 4 +TOKEN=$(curl -s -m 20 "http://localhost:$KCPORT/realms/$REALM/protocol/openid-connect/token" \ + -d grant_type=password -d "client_id=$ROPC_CLIENT_ID" \ + -d "username=$USER" -d "password=$PASS" -d scope=openid \ + | python3 -c 'import json,sys; print(json.load(sys.stdin).get("access_token",""))' 2>/dev/null || true) +if [[ -z "$TOKEN" ]]; then + echo "ERROR: could not get a token. Did you run 'make setup-keycloak'?" >&2 + cat /tmp/finance-kc-pf.log >&2 2>/dev/null || true + exit 1 +fi +echo " got token ($(echo -n "$TOKEN" | wc -c | tr -d ' ') chars)" + +say "[2/4] Port-forwarding the agent ..." +kubectl -n "$NS" port-forward "svc/$AGENT" "$PORT:8080" >/tmp/finance-pf.log 2>&1 & +PF=$!; trap 'kill $PF $KCPF 2>/dev/null || true' EXIT +sleep 4 + +send() { # $1=turn id, $2=text + curl -s -m "${TURN_TIMEOUT:-240}" -X POST "http://localhost:$PORT/" \ + -H 'Content-Type: application/json' -H "Authorization: Bearer $TOKEN" \ + -d "$(python3 -c 'import json,sys; print(json.dumps({"jsonrpc":"2.0","id":sys.argv[1],"method":"message/send","params":{"message":{"role":"user","parts":[{"kind":"text","text":sys.argv[2]}],"contextId":sys.argv[3]}}}))' "$1" "$2" "$CTX")" +} +reply() { python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); r=d.get("result",{}); print(r.get("status",{}).get("message",{}).get("parts",[{}])[0].get("text") or ("ERROR: "+json.dumps(d)))' "$1"; } + +say "[3/4] Turn 1 — descriptive refund, NO transaction id (expect SPARC to ask for the exact id):" +echo ' user> Refund my duplicate $450 subscription charge from last week.' +send 1 'Refund my duplicate $450 subscription charge from last week.' > /tmp/finance-t1.json +echo " agent> $(reply /tmp/finance-t1.json)" + +say "[4/4] Turn 2 — supply the exact id TX4827 (expect SPARC approve -> refund):" +echo ' user> The transaction id is TX4827. Please proceed.' +send 2 'The transaction id is TX4827. Please proceed with the refund, reason: duplicate charge.' > /tmp/finance-t2.json +echo " agent> $(reply /tmp/finance-t2.json)" + +say "SPARC verdicts recorded for this session:" +SID="$CTX" +kubectl -n "$NS" port-forward "deploy/$AGENT" 19094:9094 >/tmp/finance-pf94.log 2>&1 & +PF94=$!; trap 'kill $PF $PF94 $KCPF 2>/dev/null || true' EXIT +sleep 3 +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +curl -s -m 10 "http://localhost:19094/v1/sessions/$SID" 2>/dev/null | python3 "$SCRIPT_DIR/show-verdicts.py" +echo +echo "Done. Inspect the full pipeline with: make show-result" diff --git a/authbridge/demos/finance-sparc/scripts/host-setup.sh b/authbridge/demos/finance-sparc/scripts/host-setup.sh new file mode 100644 index 000000000..908eb5c5f --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/host-setup.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Optional network preflight for the demo — generic, no machine-specific tweaks. +# +# On healthy networks this is a pure no-op: it just checks that the kind node +# can reach the public internet (TLS), which it needs for image pulls and (when +# PROVIDER=watsonx) for the SPARC service to reach watsonx. +# +# A few links — some cellular hotspots and VPNs — silently blackhole PMTUD: they +# drop large TCP segments without sending ICMP "fragmentation needed", so TLS +# handshakes hang. The portable, vendor-neutral remedy is TCP MSS clamping. We +# apply it ONLY if the egress check fails, and ONLY on the kind node itself +# (standard iptables, no host/VM privileges). If your network is healthy you'll +# see "egress healthy" and nothing is changed. +# +# The demo's reasoning + reflection LLMs run in-cluster (see k8s/ollama.yaml), +# so no host-Ollama reachability or host networking is required. +set -euo pipefail + +CLUSTER=${KIND_CLUSTER_NAME:-kagenti} +NODE="${CLUSTER}-control-plane" +PROBE_URL=${EGRESS_PROBE_URL:-https://us-south.ml.cloud.ibm.com} +say() { printf '\033[1m%s\033[0m\n' "$*"; } + +# Probe egress from inside the node (no extra image pulls). Success = we got ANY +# HTTP response (the TLS endpoint is reachable); we do NOT require a 2xx, since a +# bare TLS endpoint often answers 4xx. Only a DNS/connect failure (empty code) +# counts as a blackhole. `curl` without -f returns 0 even on 4xx. +egress_ok() { + local code + code=$(docker exec "$NODE" sh -c \ + "curl -s -m 8 -o /dev/null -w '%{http_code}' '$PROBE_URL' 2>/dev/null" 2>/dev/null || true) + [ -n "$code" ] && [ "$code" != "000" ] +} + +clamp_node() { + for chain in POSTROUTING FORWARD OUTPUT; do + docker exec "$NODE" iptables -t mangle -C "$chain" \ + -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu 2>/dev/null \ + || docker exec "$NODE" iptables -t mangle -A "$chain" \ + -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu 2>/dev/null || true + done +} + +say "[net] checking cluster internet egress (TLS) ..." +if egress_ok; then + echo " egress healthy — no changes made." +else + echo " egress looks blackholed (likely a hotspot/VPN PMTUD issue); applying a portable TCP MSS clamp on the node ..." + clamp_node + if egress_ok; then echo " egress restored via MSS clamp."; else + echo " WARNING: egress still failing after clamp; image pulls / watsonx may be slow or fail on this network." >&2 + fi +fi diff --git a/authbridge/demos/finance-sparc/scripts/launch-abctl.sh b/authbridge/demos/finance-sparc/scripts/launch-abctl.sh new file mode 100644 index 000000000..f0a7ce2c0 --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/launch-abctl.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Launch abctl against the finance-agent's authbridge sidecar session API. +# abctl ships a Namespaces → Pods picker that spawns its own kubectl +# port-forward; this wrapper just verifies prerequisites and runs it. +set -euo pipefail + +NAMESPACE=${1:-team1} +AGENT_NAME=${2:-finance-agent} +ABCTL_BIN=${ABCTL_BIN:-/tmp/abctl-finance-sparc-demo} + +if [[ ! -x "$ABCTL_BIN" ]]; then + echo "ERROR: abctl binary not found at $ABCTL_BIN." >&2 + echo " Run \`make show-result\` (which depends on build-abctl) first." >&2 + exit 1 +fi +if ! kubectl -n "$NAMESPACE" get deploy "$AGENT_NAME" >/dev/null 2>&1; then + echo "ERROR: deployment $NAMESPACE/$AGENT_NAME not found. Run 'make demo' first." >&2 + exit 1 +fi +"$ABCTL_BIN" diff --git a/authbridge/demos/finance-sparc/scripts/patch-sparc-config.sh b/authbridge/demos/finance-sparc/scripts/patch-sparc-config.sh new file mode 100644 index 000000000..a1845f486 --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/patch-sparc-config.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# Patch the operator-rendered authbridge ConfigMap to add the SPARC +# pipeline (a2a-parser inbound + inference-parser + mcp-parser + sparc +# outbound). +# +# Usage: patch-sparc-config.sh +# +# The kagenti operator creates `authbridge-config-` when the agent +# pod is admitted (default pipeline: jwt-validation inbound, token-exchange +# outbound). This script extracts that config, merges k8s/sparc-patch.yaml +# via scripts/pipeline-merge.py, re-applies it, and blocks until the +# running authbridge process reports the matching config SHA (handles both +# the hot-reload and pod-roll convergence paths). +set -euo pipefail + +NAMESPACE=${1:-team1} +AGENT_NAME=${2:-finance-agent} +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +DEMO_DIR=$(dirname "$SCRIPT_DIR") +PATCH_FILE="$DEMO_DIR/k8s/sparc-patch.yaml" +CM_NAME="authbridge-config-$AGENT_NAME" + +# Resolve a python with PyYAML: prefer system python3, else use uv ephemerally. +if python3 -c 'import yaml' 2>/dev/null; then + PY="python3" +elif command -v uv >/dev/null 2>&1; then + PY="uv run --quiet --with pyyaml python3" +else + echo "ERROR: need PyYAML (pip3 install --user pyyaml) or uv on PATH." >&2 + exit 1 +fi + +# Render ${...} placeholders in the patch fragment (provider-tunable knobs). +# Defaults keep watsonx behavior identical to before. +SPARC_TIMEOUT_MS=${SPARC_TIMEOUT_MS:-30000} +SPARC_REFLECTOR_ENDPOINT=${SPARC_REFLECTOR_ENDPOINT:-http://sparc-service.kagenti-system.svc.cluster.local:8090} +RENDERED_PATCH=$(mktemp) +trap 'rm -f "$RENDERED_PATCH"' EXIT +sed -e "s|\${SPARC_TIMEOUT_MS}|${SPARC_TIMEOUT_MS}|g" \ + -e "s|\${SPARC_REFLECTOR_ENDPOINT}|${SPARC_REFLECTOR_ENDPOINT}|g" \ + "$PATCH_FILE" > "$RENDERED_PATCH" +PATCH_FILE="$RENDERED_PATCH" +echo "[*] SPARC reflection: endpoint=$SPARC_REFLECTOR_ENDPOINT timeout_ms=$SPARC_TIMEOUT_MS" + +if ! kubectl -n "$NAMESPACE" get configmap "$CM_NAME" >/dev/null 2>&1; then + echo "ERROR: ConfigMap $NAMESPACE/$CM_NAME not found." >&2 + echo " The operator creates it when the agent pod is admitted." >&2 + echo " Check: kubectl -n $NAMESPACE get pods -l app.kubernetes.io/name=$AGENT_NAME" >&2 + exit 1 +fi + +echo "[*] Merging SPARC additions into $CM_NAME ..." +CURRENT_YAML=$(kubectl -n "$NAMESPACE" get configmap "$CM_NAME" -o jsonpath='{.data.config\.yaml}') +MERGED_YAML=$(printf '%s' "$CURRENT_YAML" | $PY "$SCRIPT_DIR/pipeline-merge.py" "$PATCH_FILE") + +if [[ -z "$MERGED_YAML" ]]; then + echo "ERROR: merge produced empty output" >&2 + exit 1 +fi + +show_plugins() { + $PY -c ' +import yaml, sys +c = yaml.safe_load(sys.stdin) or {} +for d in ("inbound", "outbound"): + names = [p["name"] for p in c.get("pipeline", {}).get(d, {}).get("plugins", [])] + print(f" {d}: {names}") +' +} + +if [[ "$CURRENT_YAML" == "$MERGED_YAML" ]]; then + echo "[*] $CM_NAME already contains the SPARC pipeline — nothing to patch." + printf '%s' "$CURRENT_YAML" | show_plugins + exit 0 +fi + +echo "[*] Applying patched ConfigMap ..." +TMP_CONFIG=$(mktemp) +trap 'rm -f "$TMP_CONFIG" "$RENDERED_PATCH"' EXIT +printf '%s' "$MERGED_YAML" >"$TMP_CONFIG" +kubectl -n "$NAMESPACE" create configmap "$CM_NAME" \ + --from-file=config.yaml="$TMP_CONFIG" --dry-run=client -o yaml | kubectl apply -f - + +echo "[*] Patched. Active plugins now:" +printf '%s' "$MERGED_YAML" | show_plugins + +# Block until the running authbridge process reports the SHA we applied. +WANT_SHA=$(printf '%s' "$MERGED_YAML" | sha256sum | awk '{print $1}') +TIMEOUT=${RELOAD_TIMEOUT:-180} +DEADLINE=$(( $(date +%s) + TIMEOUT )) +echo "[*] Waiting for authbridge to load the patched config (timeout ${TIMEOUT}s)" +echo " target SHA: $WANT_SHA" +ACTIVE_SHA="" +while [[ $(date +%s) -lt $DEADLINE ]]; do + ACTIVE_SHA=$(kubectl -n "$NAMESPACE" exec deploy/"$AGENT_NAME" -c authbridge-proxy -- \ + wget -q -O - http://localhost:9093/reload/status 2>/dev/null | \ + python3 -c 'import json,sys +try: print(json.load(sys.stdin).get("active_config_sha256","")) +except Exception: pass' 2>/dev/null || true) + if [[ "$ACTIVE_SHA" == "$WANT_SHA" ]]; then + echo "[*] Active config SHA matches — SPARC pipeline is live." + exit 0 + fi + sleep 3 +done + +echo "ERROR: authbridge active config did not match patched SHA within ${TIMEOUT}s." >&2 +echo " want: $WANT_SHA" >&2 +echo " last active: ${ACTIVE_SHA:-}" >&2 +kubectl -n "$NAMESPACE" logs deploy/"$AGENT_NAME" -c authbridge-proxy --tail=20 >&2 || true +exit 1 diff --git a/authbridge/demos/finance-sparc/scripts/pipeline-merge.py b/authbridge/demos/finance-sparc/scripts/pipeline-merge.py new file mode 100644 index 000000000..fdf97703b --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/pipeline-merge.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Merge pipeline additions into the operator-rendered authbridge config.yaml. + +Generic over the plugin set: the patch file (argv[1]) declares +`inbound_prepend` / `outbound_append` lists of plugin entries; this merges +them into the operator's config (read from stdin) and writes the result to +stdout. Idempotent — entries matched by `name` are not duplicated. + +(Identical in shape to demos/ibac/scripts/ibac-merge.py; kept demo-local so +the finance-sparc demo is self-contained.) +""" + +import sys + +import yaml + + +def main() -> int: + if len(sys.argv) != 2: + sys.stderr.write("usage: pipeline-merge.py \n") + return 2 + + operator = yaml.safe_load(sys.stdin) or {} + with open(sys.argv[1]) as f: + patch = yaml.safe_load(f) or {} + + pipeline = operator.setdefault("pipeline", {}) + in_plugins = pipeline.setdefault("inbound", {}).setdefault("plugins", []) + out_plugins = pipeline.setdefault("outbound", {}).setdefault("plugins", []) + in_names = {p.get("name") for p in in_plugins} + out_names = {p.get("name") for p in out_plugins} + + # Reverse-then-prepend preserves the patch's natural order at the front. + for entry in reversed(patch.get("inbound_prepend", []) or []): + if entry.get("name") not in in_names: + in_plugins.insert(0, entry) + in_names.add(entry["name"]) + + for entry in patch.get("outbound_append", []) or []: + if entry.get("name") not in out_names: + out_plugins.append(entry) + out_names.add(entry["name"]) + + sys.stdout.write(yaml.safe_dump(operator, default_flow_style=False, sort_keys=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/authbridge/demos/finance-sparc/scripts/setup_keycloak_finance.py b/authbridge/demos/finance-sparc/scripts/setup_keycloak_finance.py new file mode 100644 index 000000000..2aafc1b0b --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/setup_keycloak_finance.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Keycloak setup for the finance-sparc demo. + +Makes the finance-agent's inbound jwt-validation pass for tokens minted by the +demo (scripted ROPC) and by the kagenti UI, by creating an audience client +scope that puts the agent's SPIFFE id in the token `aud`, and a public +direct-access (ROPC) client + a demo user `alice`. + +Idempotent. Run via: uv run --with python-keycloak python setup_keycloak_finance.py +Env overrides: KEYCLOAK_URL, KEYCLOAK_REALM, KEYCLOAK_ADMIN_USERNAME/PASSWORD, +UI_CLIENT_ID, NAMESPACE, AGENT_SA. +""" +import os +import sys + +from keycloak import KeycloakAdmin + +KEYCLOAK_URL = os.environ.get("KEYCLOAK_URL", "http://keycloak.localtest.me:8080") +KEYCLOAK_REALM = os.environ.get("KEYCLOAK_REALM", "kagenti") +ADMIN_USER = os.environ.get("KEYCLOAK_ADMIN_USERNAME", "admin") +ADMIN_PASS = os.environ.get("KEYCLOAK_ADMIN_PASSWORD", "admin") +UI_CLIENT_ID = os.environ.get("UI_CLIENT_ID", "kagenti") +SPIFFE_TRUST_DOMAIN = os.environ.get("SPIFFE_TRUST_DOMAIN", "localtest.me") +NAMESPACE = os.environ.get("NAMESPACE", "team1") +AGENT_SA = os.environ.get("AGENT_SA", "finance-agent") +ROPC_CLIENT_ID = os.environ.get("ROPC_CLIENT_ID", "finance-sparc-e2e") +USER_NAME = "alice" +USER_PASS = "alice123" +USER = {"username": USER_NAME, "email": "alice@example.com", + "enabled": True, "emailVerified": True, "firstName": "Alice", "lastName": "Demo"} + + +def main() -> int: + agent_spiffe = f"spiffe://{SPIFFE_TRUST_DOMAIN}/ns/{NAMESPACE}/sa/{AGENT_SA}" + scope_name = f"agent-{NAMESPACE}-{AGENT_SA}-aud" + print(f"Keycloak: {KEYCLOAK_URL} realm={KEYCLOAK_REALM} agent aud={agent_spiffe}") + + kc = KeycloakAdmin(server_url=KEYCLOAK_URL, username=ADMIN_USER, password=ADMIN_PASS, + realm_name=KEYCLOAK_REALM, user_realm_name="master") + + # 1) audience client scope + mapper → agent SPIFFE id in aud + scope_id = next((s["id"] for s in kc.get_client_scopes() if s["name"] == scope_name), None) + if not scope_id: + scope_id = kc.create_client_scope({ + "name": scope_name, "protocol": "openid-connect", + "attributes": {"include.in.token.scope": "true", "display.on.consent.screen": "false"}, + }, skip_exists=True) + print(f" client scope {scope_name} -> {scope_id}") + try: + kc.add_mapper_to_client_scope(scope_id, { + "name": scope_name + "-aud-mapper", "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "config": {"included.custom.audience": agent_spiffe, + "id.token.claim": "false", "access.token.claim": "true"}, + }) + except Exception as e: + print(f" (mapper exists or: {e})") + try: + kc.add_default_default_client_scope(scope_id) + print(f" '{scope_name}' is now a realm default scope") + except Exception as e: + print(f" (realm default: {e})") + + # 2) public ROPC client for scripted drive + if not kc.get_client_id(ROPC_CLIENT_ID): + kc.create_client({"clientId": ROPC_CLIENT_ID, "name": "finance-sparc E2E (direct access)", + "enabled": True, "publicClient": True, "standardFlowEnabled": True, + "directAccessGrantsEnabled": True}, skip_exists=True) + ropc_internal = kc.get_client_id(ROPC_CLIENT_ID) + for label, cid in [("ROPC " + ROPC_CLIENT_ID, ropc_internal), + ("UI " + UI_CLIENT_ID, kc.get_client_id(UI_CLIENT_ID))]: + if cid: + try: + kc.add_client_default_client_scope(cid, scope_id, {}) + print(f" added aud scope as default on {label}") + except Exception as e: + print(f" ({label} default scope: {e})") + else: + print(f" note: client {label} not found (ok)") + + # 3) demo user alice (admin role → backend operator role for /api/chat) + uid = kc.get_user_id(USER_NAME) + if not uid: + uid = kc.create_user(USER, exist_ok=True) + kc.set_user_password(uid, USER_PASS, temporary=False) + try: + kc.assign_realm_roles(uid, [kc.get_realm_role("admin")]) + except Exception as e: + print(f" (assign admin role: {e})") + print(f" user alice ready ({uid})") + + print("Keycloak setup complete.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/authbridge/demos/finance-sparc/scripts/show-verdicts.py b/authbridge/demos/finance-sparc/scripts/show-verdicts.py new file mode 100644 index 000000000..d089551db --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/show-verdicts.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Print the SPARC plugin verdicts from a session snapshot (read on stdin).""" +import json +import sys + +try: + data = json.load(sys.stdin) +except Exception: + print(" (no session events yet)") + sys.exit(0) + +seen = False +for event in data.get("events", []): + for inv in (event.get("invocations") or {}).get("outbound", []): + if inv.get("plugin") == "sparc" and inv.get("action") in ("allow", "modify", "deny", "observe"): + det = inv.get("details", {}) + seen = True + score = det.get("score") + score_s = f" score={score}" if score not in (None, "") else "" + print( + " SPARC {}/{} tool={}{}".format( + inv.get("action"), inv.get("reason"), det.get("tool"), score_s + ) + ) +if not seen: + print(" (no sparc verdicts found in session; check `make logs-sparc`)") diff --git a/authbridge/demos/finance-sparc/scripts/stage-ollama-model.sh b/authbridge/demos/finance-sparc/scripts/stage-ollama-model.sh new file mode 100644 index 000000000..54bd13fb4 --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/stage-ollama-model.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Stage an Ollama model onto the kind node for the in-cluster Ollama (mounted at +# /opt/ollama-models via hostPath). Copies the model from the host's local +# Ollama store with `docker cp` (local, no network). If the model isn't present +# on the host, the in-cluster Ollama will pull it on first use instead. +# +# Usage: stage-ollama-model.sh [model] (default llama3.2:3b) +set -euo pipefail + +MODEL=${1:-${OLLAMA_MODEL:-llama3.2:3b}} +CLUSTER=${KIND_CLUSTER_NAME:-kagenti} +NODE="${CLUSTER}-control-plane" +STORE="${OLLAMA_MODELS_DIR:-$HOME/.ollama/models}" +REPO="${MODEL%%:*}"; TAG="${MODEL##*:}" +# Ollama library models live under registry.ollama.ai/library//. +case "$REPO" in */*) NS_PATH="$REPO" ;; *) NS_PATH="library/$REPO" ;; esac +MAN="$STORE/manifests/registry.ollama.ai/$NS_PATH/$TAG" + +if [[ ! -f "$MAN" ]]; then + echo "[stage] model $MODEL not found in host store ($MAN)." + echo "[stage] the in-cluster Ollama will pull it on first use (needs egress)." + exit 0 +fi + +echo "[stage] copying $MODEL from host store to node $NODE:/opt/ollama-models ..." +docker exec "$NODE" mkdir -p "/opt/ollama-models/manifests/registry.ollama.ai/$NS_PATH" /opt/ollama-models/blobs +docker cp "$MAN" "$NODE:/opt/ollama-models/manifests/registry.ollama.ai/$NS_PATH/$TAG" +for d in $(python3 -c "import json;m=json.load(open('$MAN'));print(m['config']['digest']);[print(l['digest']) for l in m['layers']]"); do + blob="sha256-${d#sha256:}" + if ! docker exec "$NODE" test -s "/opt/ollama-models/blobs/$blob" 2>/dev/null; then + docker cp "$STORE/blobs/$blob" "$NODE:/opt/ollama-models/blobs/$blob" + fi +done +echo "[stage] done: $(docker exec "$NODE" du -sh /opt/ollama-models | awk '{print $1}') staged for $MODEL" diff --git a/authbridge/docs/sparc-plugin.md b/authbridge/docs/sparc-plugin.md new file mode 100644 index 000000000..8c4909089 --- /dev/null +++ b/authbridge/docs/sparc-plugin.md @@ -0,0 +1,165 @@ +# SPARC (Pre-Tool Reflection) Plugin + +The `sparc` plugin verifies that an agent's proposed tool call is *grounded* in the +conversation and the available tool specifications before it executes — catching +**hallucinated / ungrounded arguments** (e.g. an invented transaction id) and inappropriate +tool selection. On a reflected reject it returns SPARC's clarification to the agent so the +agent re-asks the user; the bad call never runs. + +SPARC is the `SPARCReflectionComponent` from the +[`agent-lifecycle-toolkit`](https://pypi.org/project/agent-lifecycle-toolkit/) (ALTK) Python +package, served over HTTP by the companion [SPARC reflection service](../sparc-service/README.md). +Because AuthBridge plugins are Go, this plugin calls the service over HTTP (the same shape +`ibac` uses for its judge). All enforcement policy lives in the plugin; the service only +returns SPARC's verdict. + +It is **complementary to [`ibac`](ibac-plugin.md)**: SPARC verifies argument *grounding*; +IBAC verifies *intent alignment* (prompt-injection / exfiltration). + +## Generic by design — works for any agent + +SPARC's three inputs are collected from exactly what the agent produces, with no per-agent +wiring or out-of-band data: + +- **conversation history, including the system prompt** — from the agent's LLM call captured + by `inference-parser` (`pctx.Extensions.Inference.Messages`, every role preserved); +- **tool specifications in OpenAI function-calling format** — from the same call + (`pctx.Extensions.Inference.Tools`); +- **the proposed tool call** — from the outbound MCP `tools/call` (`mcp` mode) or from the LLM + response (`inference` mode). + +If the conversation/tool context isn't available (no `inference-parser`, or session tracking +off), the plugin **skips** (`no_inference_context`) rather than reflect on partial data. + +## Enforcement is format-aware + +The verdict is returned to the agent in the shape it expects, selected by `enforcement`: + +- **`mcp` (default — the kagenti norm).** Gate the outbound MCP `tools/call`. The tool call + comes from the MCP request; conversation + tool specs are correlated from the session's most + recent inference request (robust to LLM streaming). On a reflected reject, return SPARC's + clarification as a **JSON-RPC MCP tool result** (`Reject` + `Violation{Status:200, Body}`); + the agent's MCP client consumes it like any tool output. +- **`inference` (for non-MCP agents).** Gate the agent's LLM response, where all three inputs + are co-located. On a reflected reject, rewrite the completion (via `SetResponseBody`) so the + assistant turn carries the clarification and the tool_call is dropped — norms-correct for any + OpenAI-style agent. + +## On-reject policy + +When SPARC returns `reject`, `on_reject_action` decides, with optional score escalation: + +| `on_reject_action` | Behavior | Session record | +|---|---|---| +| `observe` | Log only, let the call through (shadow). | `observe` / `reject_observed` | +| `reflect` (default) | Return SPARC's clarification to the agent (MCP result or rewritten completion). | `modify` / `reflected` | +| `deny` | Hard block (403 in `mcp` mode; refusal completion in `inference` mode). | `deny` / `blocked` | + +When `deny_score_threshold > 0` and SPARC's `overall_avg_score` (normalized 0..1, higher = better +grounded) is `<=` it, any reject is escalated to `deny`. + +> **`fail_policy` defaults to `open`** — when SPARC is unreachable, times out, or returns +> `decision=error`, the proposed tool call is allowed through (and recorded). This is deliberate +> and **diverges from `ibac`, which fails closed**: SPARC is a grounding *quality gate*, not an +> auth control, so reflector downtime shouldn't take the agent's tools offline. Set +> `fail_policy: closed` when you want tool execution gated on reflector availability (e.g. a +> high-assurance deployment where an unverified call is worse than a failed one). The same policy +> governs the case where the conversation/tool context is missing (no `inference-parser` event): +> `open` skips and records `no_inference_context`; `closed` blocks. + +## Configuration + +```yaml +pipeline: + outbound: + plugins: + - name: sparc + config: + reflector_endpoint: "http://sparc-service.kagenti-system.svc:8090" + enforcement: "mcp" # mcp | inference + track: "fast_track" # fast_track|slow_track|syntax|spec_free|transformations_only + on_reject_action: "reflect" # observe | reflect | deny + deny_score_threshold: 0 # 0 disables; e.g. 2.0 → deny rejects scoring <= 2 + fail_policy: "open" # open=allow+log on SPARC error; closed=block + timeout_ms: 30000 + skip_tools: ["list_*"] # tool-name globs NOT reflected on (e.g. read-only tools) + reflect_tools: [] # if non-empty, ONLY these globs are reflected +``` + +| Field | Required | Default | Description | +|---|---|---|---| +| `reflector_endpoint` | Yes | — | Base URL of the SPARC service; the plugin POSTs to `{endpoint}/reflect`. | +| `reflector_bearer` | No | `""` | Bearer for the service. Empty for in-cluster unauthenticated calls. | +| `enforcement` | No | `mcp` | `mcp` (gate MCP tools/call, return MCP result) or `inference` (gate LLM response, rewrite completion). | +| `track` | No | `fast_track` | SPARC track. `fast_track` = hallucination + function-selection checks. | +| `on_reject_action` | No | `reflect` | `observe` \| `reflect` \| `deny` (see table above). | +| `deny_score_threshold` | No | `0` | Escalate a reject to `deny` when score `<=` this (`[0,1]`; `0` disables). | +| `fail_policy` | No | `open` | On SPARC unreachable / `decision=error`: `open` allows+logs; `closed` blocks. | +| `timeout_ms` | No | `30000` | Per-call reflection timeout. Rejected below `100`. Reflection is **out-of-band** (the plugin calls the SPARC service directly), so this is independent of the forward proxy's upstream timeout — raise it for slow/CPU-bound providers (e.g. an in-cluster Ollama). | +| `skip_tools` | No | — | Tool-name globs (`path.Match`) to NOT reflect on (e.g. trivial read tools). | +| `reflect_tools` | No | — | If non-empty, ONLY reflect tools matching these globs; all others skipped. | +| `bypass_hosts` / `bypass_paths` | No | built-in | Host / path globs skipped without reflecting. | + +### Pipeline composition + +`sparc` declares `RequiresAny: ["inference-parser", "mcp-parser"]`. For the richest, generic +reflection wire all three parsers: + +```yaml +pipeline: + inbound: + plugins: + - name: a2a-parser # seeds the session with the user turn + - name: jwt-validation + outbound: + plugins: + - name: token-exchange + - name: inference-parser # captures messages (incl. system) + tool specs + - name: mcp-parser # parses the tool call (mcp mode) + - name: sparc +``` + +## Observability + +`sparc` records a flat `Invocation` per call (action `allow`/`modify`/`deny`/`observe`, +`reason`, `tool`, `score`) **and** publishes the full structured verdict via the plugin-event +escape-hatch (`Extensions.Custom["sparc/event"]` → `SessionEvent.Plugins["sparc"]`), so abctl +and other consumers can render the decision, score, track, enforcement mode, and per-issue +explanations/corrections. + +## Status codes & reasons + +| Reason | HTTP / effect | Meaning | +|---|---|---| +| `sparc.reflected` | 200 MCP result / rewritten completion | A reject was reflected back to the agent as a clarification. | +| `sparc.blocked` | 403 (mcp) / refusal completion (inference) | A reject handled by `deny` (or escalated via score). | +| `sparc.reflector_unavailable` | 503 / refusal (closed); pass-through (open) | SPARC unreachable or `decision=error`, per `fail_policy`. | +| `sparc.no_inference_context` | 503 (closed); pass-through (open) | No conversation/tool context to ground against, per `fail_policy`. | + +## Security / network posture + +The reflection service is an **in-cluster backend** — same trust model as the authbridge session +API (`:9094`): no ingress, never exposed publicly. Its `/reflect` endpoint is unauthenticated by +default and runs outside the agent-side ambient mesh (`kagenti.io/inject: disabled`), so it makes +its own egress to the LLM provider. Treat the cluster network boundary as the control: restrict +who can reach it with a `NetworkPolicy` (allow only the authbridge sidecars), since any pod that +can POST to `/reflect` can trigger LLM calls billed to the configured credentials. For an extra +hop, set `reflector_bearer` on the plugin and terminate it at the service (or a sidecar). The +service never echoes provider exception text back to callers; details stay in its logs. + +## Limitations & non-goals +- Not an intent/auth control — use `ibac` + `token-exchange` routes / NetworkPolicy for that. +- No principal/identity input — SPARC reflects on `(messages, tool_specs, tool_calls)` only. +- `inference` mode can't see streamed tool_calls (use `mcp` mode for streaming MCP agents). +- Cost: one reflection round-trip per (non-skipped) tool call. + +## See also +- [`sparc-service/README.md`](../sparc-service/README.md) — the reflection service. +- [`demos/finance-sparc/`](../demos/finance-sparc/README.md) — the end-to-end demo. +- [`ibac-plugin.md`](ibac-plugin.md) — the complementary intent control. + +## Files +- `authlib/plugins/sparc/plugin.go` — config, gates, policy, structured event. +- `authlib/plugins/sparc/collect.go` — generic input collection. +- `authlib/plugins/sparc/respond.go` — MCP result + completion-rewrite responders. +- `authlib/plugins/sparc/reflector.go` — HTTP client to the service. diff --git a/authbridge/sparc-service/.dockerignore b/authbridge/sparc-service/.dockerignore new file mode 100644 index 000000000..b1bce9d9d --- /dev/null +++ b/authbridge/sparc-service/.dockerignore @@ -0,0 +1,7 @@ +.venv +__pycache__ +*.pyc +tests +.pytest_cache +*.egg-info +.env diff --git a/authbridge/sparc-service/Dockerfile b/authbridge/sparc-service/Dockerfile new file mode 100644 index 000000000..fa504deb7 --- /dev/null +++ b/authbridge/sparc-service/Dockerfile @@ -0,0 +1,24 @@ +# Kagenti SPARC reflection service. +# Depends on the agent-lifecycle-toolkit (ALTK) package straight from PyPI — no vendoring. +FROM python:3.12-slim AS base + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + LITELLM_LOCAL_MODEL_COST_MAP=True + +WORKDIR /app + +# Install dependencies first for better layer caching. +COPY pyproject.toml README.md ./ +COPY sparc_service ./sparc_service +RUN pip install --upgrade pip && pip install . + +# Drop privileges. +RUN useradd --create-home --uid 10001 sparc +USER sparc + +EXPOSE 8090 + +# PORT/HOST and SPARC_* / WX_* are provided via the Deployment env (ConfigMap/Secret). +CMD ["python", "-m", "sparc_service"] diff --git a/authbridge/sparc-service/README.md b/authbridge/sparc-service/README.md new file mode 100644 index 000000000..93b98c2e3 --- /dev/null +++ b/authbridge/sparc-service/README.md @@ -0,0 +1,136 @@ +# Kagenti SPARC reflection service + +A thin, in-process HTTP wrapper around the **SPARC** pre-tool reflection component +(`altk.pre_tool.sparc.SPARCReflectionComponent`) from the +[`agent-lifecycle-toolkit`](https://pypi.org/project/agent-lifecycle-toolkit/) (ALTK) +PyPI package. + +AuthBridge's Go [`sparc` plugin](../authlib/plugins/sparc) calls this service to decide +whether a proposed tool call is **grounded** in the conversation and the available tool +specs. SPARC catches hallucinated / ungrounded arguments (e.g. an invented transaction +ID) and inappropriate function selection. + +This service is deliberately *just* a reflection wrapper: + +- It consumes ALTK **directly from PyPI** — no vendored copy of the SPARC source. +- It calls SPARC **in process** — no relay queue, observer, or host worker (those exist + only in the original `ibac` reference repo as a `kind`-egress workaround). +- It returns SPARC's verdict **faithfully**. All enforcement policy (observe / inject / + deny, score thresholds) lives in the AuthBridge plugin, not here. + +## API + +### `POST /reflect` + +Request: + +```jsonc +{ + "messages": [ { "role": "user", "content": "Refund transaction TX482." } ], + "tool_specs": [ { "type": "function", "function": { "name": "get_transaction", ... } } ], + "tool_calls": [ { "id": "c1", "type": "function", + "function": { "name": "get_transaction", + "arguments": "{\"transaction_id\": \"TX4821\"}" } } ], + "session_id": "optional-correlation-id", + "track": "fast_track" // optional per-request override +} +``` + +Response: + +```jsonc +{ + "decision": "approve | reject | error", + "issues": [ { "issue_type": "...", "metric_name": "...", + "explanation": "...", "correction": { ... } } ], + "overall_avg_score": 0.4, // SPARC grounding score, normalized 0..1 (higher = better), when available + "execution_time_ms": 4600.0, + "raw_pipeline_result": { ... } // present only if SPARC_INCLUDE_RAW_RESPONSE=true +} +``` + +### `GET /healthz` — liveness. `GET /readyz` — readiness (config valid + component buildable). + +On failure `/reflect` returns a stable message (`{"error": "reflection failed"}`, or `400` for a +bad request such as an unsupported `track`); the underlying provider exception is logged, never +returned to the caller — provider errors can embed endpoints or credentials. + +## Security / network posture + +Deploy this as an **in-cluster backend only** — no ingress, never public. `/reflect` is +unauthenticated by default, so the network boundary is the control: restrict callers with a +`NetworkPolicy` (allow only the authbridge sidecars). Any pod that can POST to `/reflect` can +trigger LLM calls billed to the configured credentials. For an additional hop, set the plugin's +`reflector_bearer` and verify it here (or at a fronting sidecar). The service runs as a non-root +user and makes its own egress to the LLM provider; it stays out of the agent-side ambient mesh. + +## Configuration (environment) + +**watsonx is the default.** Switching providers is a config change — no rebuild. `watsonx` +and `ollama` use ALTK's provider-native clients; `openai`, `azure` (Azure OpenAI), and the +generic `litellm` provider use ALTK's generic LiteLLM client, so **any provider LiteLLM/ALTK +supports** works by setting the model string and (optionally) extra client kwargs. + +| Variable | Default | Notes | +| --- | --- | --- | +| `SPARC_LLM_PROVIDER` | `watsonx` | `watsonx` \| `ollama` \| `openai` \| `azure` \| `litellm` (generic) | +| `SPARC_MODEL` | per-provider | watsonx: `mistral-large-2512`, ollama: `llama3.2:3b`, openai: `gpt-4o-mini`. **Required** for `azure`/`litellm` (e.g. `azure/`, `anthropic/claude-3-5-sonnet`, `gemini/gemini-1.5-pro`, `bedrock/...`). | +| `SPARC_LLM_KWARGS_JSON` | — | JSON object of extra LiteLLM client kwargs for `openai`/`azure`/`litellm`, e.g. `{"api_base":"...","api_version":"...","api_key":"..."}`. | +| `SPARC_LLM_REGISTRY_ID` | — | Advanced: override the ALTK client registry id directly (any ALTK client). | +| `SPARC_TRACK` | `fast_track` | `fast_track` \| `slow_track` \| `syntax` \| `spec_free` \| `transformations_only` | +| `SPARC_LLM_TIMEOUT` | `120` | seconds | +| `SPARC_RETRIES` | `3` | SPARC retries | +| `SPARC_MAX_PARALLEL` | `2` | SPARC metric parallelism | +| `SPARC_INCLUDE_RAW_RESPONSE` | `true` | include `raw_pipeline_result` in responses | +| `WX_API_KEY` / `WX_PROJECT_ID` / `WX_URL` | — | watsonx creds (also accepts `WATSONX_*`) | +| `OLLAMA_BASE_URL` | `http://localhost:11434` | ollama endpoint | +| `OPENAI_API_KEY` / `OPENAI_BASE_URL` | — | openai creds (LiteLLM also reads provider keys from env: `OPENAI_API_KEY`, `AZURE_API_KEY`, `ANTHROPIC_API_KEY`, …) | +| `HOST` / `PORT` | `0.0.0.0` / `8090` | bind address | + +### Provider examples + +```bash +# watsonx (default) +SPARC_LLM_PROVIDER=watsonx WX_API_KEY=... WX_PROJECT_ID=... + +# OpenAI +SPARC_LLM_PROVIDER=openai SPARC_MODEL=gpt-4o-mini OPENAI_API_KEY=sk-... + +# Azure OpenAI +SPARC_LLM_PROVIDER=azure SPARC_MODEL=azure/ \ + SPARC_LLM_KWARGS_JSON='{"api_base":"https://.openai.azure.com","api_version":"2024-06-01","api_key":"..."}' + +# Anything else LiteLLM supports (Anthropic, Gemini, Bedrock, ...) +SPARC_LLM_PROVIDER=litellm SPARC_MODEL=anthropic/claude-3-5-sonnet \ + SPARC_LLM_KWARGS_JSON='{"api_key":"sk-ant-..."}' + +# Local Ollama +SPARC_LLM_PROVIDER=ollama SPARC_MODEL=llama3.2:3b OLLAMA_BASE_URL=http://host:11434 +``` + +## Develop & test + +```bash +python -m venv .venv && . .venv/bin/activate +pip install -e ".[dev]" + +# Fast, offline unit tests (fake SPARC component — no network, no creds): +pytest + +# Opt-in integration test against real watsonx: +RUN_WATSONX_TESTS=1 WX_API_KEY=... WX_PROJECT_ID=... pytest tests/test_watsonx_integration.py -v +``` + +## Run + +```bash +SPARC_LLM_PROVIDER=watsonx WX_API_KEY=... WX_PROJECT_ID=... python -m sparc_service +# or against local Ollama: +SPARC_LLM_PROVIDER=ollama SPARC_MODEL=llama3.2:3b python -m sparc_service +``` + +## Build the image + +```bash +docker build -t kagenti-sparc-service:latest . +``` diff --git a/authbridge/sparc-service/pyproject.toml b/authbridge/sparc-service/pyproject.toml new file mode 100644 index 000000000..fdae7b124 --- /dev/null +++ b/authbridge/sparc-service/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "kagenti-sparc-service" +version = "0.1.0" +description = "Kagenti SPARC reflection service — a thin in-process HTTP wrapper around the SPARC pre-tool reflection component from the agent-lifecycle-toolkit (ALTK)." +readme = "README.md" +requires-python = ">=3.10" +license = { text = "Apache-2.0" } +dependencies = [ + # The SPARC engine itself, consumed directly from PyPI (no vendoring). + "agent-lifecycle-toolkit==0.10.1", + "fastapi>=0.110", + "uvicorn[standard]>=0.27", + "pydantic>=2", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8", + "httpx>=0.27", +] + +[project.scripts] +kagenti-sparc-service = "sparc_service.__main__:main" + +[tool.hatch.build.targets.wheel] +packages = ["sparc_service"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" diff --git a/authbridge/sparc-service/sparc_service/__init__.py b/authbridge/sparc-service/sparc_service/__init__.py new file mode 100644 index 000000000..afbf6b7b3 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/__init__.py @@ -0,0 +1,16 @@ +"""Kagenti SPARC reflection service. + +A thin, in-process HTTP wrapper around the SPARC pre-tool reflection component +(`altk.pre_tool.sparc.SPARCReflectionComponent`) shipped in the +``agent-lifecycle-toolkit`` PyPI package. + +The service exposes a single reflection endpoint that AuthBridge's Go ``sparc`` +plugin calls to decide whether a proposed tool call is grounded in the +conversation and tool specifications. All enforcement policy (observe / inject / +deny, score thresholds) lives in the plugin; this service only returns SPARC's +verdict faithfully. +""" + +__all__ = ["__version__"] + +__version__ = "0.1.0" diff --git a/authbridge/sparc-service/sparc_service/__main__.py b/authbridge/sparc-service/sparc_service/__main__.py new file mode 100644 index 000000000..a0f387a4d --- /dev/null +++ b/authbridge/sparc-service/sparc_service/__main__.py @@ -0,0 +1,16 @@ +"""Console/`python -m sparc_service` entry point.""" + +from __future__ import annotations + +import uvicorn + +from .settings import Settings + + +def main() -> None: + settings = Settings.from_env() + uvicorn.run("sparc_service.api:app", host=settings.host, port=settings.port, log_level="info") + + +if __name__ == "__main__": + main() diff --git a/authbridge/sparc-service/sparc_service/api.py b/authbridge/sparc-service/sparc_service/api.py new file mode 100644 index 000000000..3a7552083 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/api.py @@ -0,0 +1,70 @@ +"""FastAPI application for the SPARC reflection service. + +Endpoints: + POST /reflect — run SPARC on a proposed tool call, return the verdict. + GET /healthz — liveness (always ok if the process is up). + GET /readyz — readiness (config valid and component buildable). +""" + +from __future__ import annotations + +import logging + +from fastapi import FastAPI, HTTPException +from fastapi.concurrency import run_in_threadpool + +from .engine import ReflectionEngine +from .models import ReflectRequest, ReflectResponse +from .settings import Settings + +log = logging.getLogger(__name__) + + +def create_app(engine: ReflectionEngine | None = None) -> FastAPI: + """Build the FastAPI app. Inject ``engine`` in tests; defaults to env config.""" + settings = engine.settings if engine is not None else Settings.from_env() + engine = engine or ReflectionEngine(settings) + + app = FastAPI( + title="Kagenti SPARC reflection service", + version="0.1.0", + summary="In-process SPARC pre-tool reflection over HTTP.", + ) + app.state.engine = engine + app.state.settings = settings + + @app.get("/healthz") + def healthz() -> dict[str, object]: + return { + "status": "ok", + "provider": settings.provider, + "model": settings.model, + "track": settings.track, + } + + @app.get("/readyz") + def readyz() -> dict[str, object]: + ok, detail = engine.ready() + if not ok: + raise HTTPException(status_code=503, detail={"status": "not_ready", "reason": detail}) + return {"status": "ready", "provider": settings.provider, "model": settings.model} + + @app.post("/reflect", response_model=ReflectResponse) + async def reflect(request: ReflectRequest) -> ReflectResponse: + # SPARCReflectionComponent.process is synchronous (and CPU/IO bound on the + # LLM call); run it off the event loop so the service stays responsive. + try: + return await run_in_threadpool(engine.reflect, request) + except ValueError as exc: # bad input (e.g. unsupported track) → 400 + raise HTTPException(status_code=400, detail={"error": str(exc)}) from exc + except Exception: + # Reflection failure → 502 so the plugin can apply its fail policy. + # Keep the full error in the service logs; never echo provider + # exception text to the caller — it can embed endpoints/credentials. + log.exception("reflection failed") + raise HTTPException(status_code=502, detail={"error": "reflection failed"}) from None + + return app + + +app = create_app() diff --git a/authbridge/sparc-service/sparc_service/engine.py b/authbridge/sparc-service/sparc_service/engine.py new file mode 100644 index 000000000..053fbf929 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/engine.py @@ -0,0 +1,136 @@ +"""The reflection engine: lazily builds SPARC components and maps their output. + +Components are built lazily and cached *per track* so the service starts even +when the LLM backend is briefly unavailable, switches tracks per request without +rebuilding the common case, and lets unit tests inject a fake component factory +(no network, no credentials). +""" + +from __future__ import annotations + +import logging +import threading +from dataclasses import replace +from typing import Any, Callable + +from .models import ReflectionIssue, ReflectRequest, ReflectResponse +from .providers import build_component +from .settings import SUPPORTED_TRACKS, Settings + +log = logging.getLogger(__name__) + +# A factory takes a track name and returns an object exposing +# ``process(run_input, phase) -> output`` where output has +# ``reflection_result``, ``execution_time_ms`` and ``raw_pipeline_result``. +# The real one is altk's SPARCReflectionComponent. +ComponentFactory = Callable[[str], Any] + + +def _decision_str(decision: Any) -> str: + """Normalize SPARC's decision enum/str to a plain string.""" + return str(getattr(decision, "value", decision)) + + +def _extract_overall_score(raw_pipeline: dict[str, Any] | None) -> float | None: + if not raw_pipeline: + return None + value = raw_pipeline.get("overall_avg_score") + return float(value) if isinstance(value, (int, float)) else None + + +class ReflectionEngine: + """Builds (per track) and invokes SPARC components, mapping IO to wire models.""" + + def __init__( + self, + settings: Settings, + component_factory: ComponentFactory | None = None, + ) -> None: + self._settings = settings + self._factory: ComponentFactory = component_factory or ( + lambda track: build_component(replace(settings, track=track)) + ) + self._lock = threading.Lock() + # Components keyed by track, so a per-request track override doesn't + # rebuild the common case. Most deployments use exactly one track. + self._components: dict[str, Any] = {} + + @property + def settings(self) -> Settings: + return self._settings + + def _component_or_build(self, track: str) -> Any: + with self._lock: + existing = self._components.get(track) + if existing is not None: + return existing + component = self._factory(track) # may raise → surfaced as 503 / not-ready + self._components[track] = component + return component + + def ready(self) -> tuple[bool, str | None]: + """Best-effort readiness: try to build the default-track component once. + + Our own validation errors (missing creds, etc.) are safe to surface + verbatim. A build exception may carry provider text (endpoints, request + details), so it's logged in full and reported as a generic reason. + """ + if self._settings.errors: + return False, "; ".join(self._settings.errors) + try: + self._component_or_build(self._settings.track) + return True, None + except Exception: + log.exception("SPARC component build failed") + return False, "component initialization failed; see service logs" + + def reflect(self, request: ReflectRequest) -> ReflectResponse: + """Run SPARC reflection on a proposed tool call and project the verdict.""" + from altk.core.toolkit import AgentPhase + from altk.pre_tool.core import SPARCReflectionRunInput + + track = request.track or self._settings.track + if track not in SUPPORTED_TRACKS: + raise ValueError(f"unsupported track {track!r}; expected one of {sorted(SUPPORTED_TRACKS)}") + + component = self._component_or_build(track) + + run_input = SPARCReflectionRunInput( + messages=request.messages, + tool_specs=request.tool_specs, + tool_calls=request.tool_calls, + ) + result = component.process(run_input, phase=AgentPhase.RUNTIME) + output = result.output + reflection = output.reflection_result + raw_pipeline = getattr(output, "raw_pipeline_result", None) or {} + + issues = [ + ReflectionIssue( + issue_type=getattr(i, "issue_type", None), + metric_name=getattr(i, "metric_name", None), + explanation=getattr(i, "explanation", None), + correction=getattr(i, "correction", None), + ) + for i in reflection.issues + ] + + decision = _decision_str(reflection.decision) + score = _extract_overall_score(raw_pipeline) + execution_ms = getattr(output, "execution_time_ms", None) + log.info( + "reflect session=%s track=%s decision=%s score=%s ms=%s", + request.session_id or "-", + track, + decision, + f"{score:.2f}" if score is not None else "-", + f"{execution_ms:.1f}" if isinstance(execution_ms, (int, float)) else "-", + ) + + return ReflectResponse( + decision=decision, + issues=issues, + overall_avg_score=score, + execution_time_ms=execution_ms, + raw_pipeline_result=raw_pipeline if self._settings.include_raw_response else None, + ) diff --git a/authbridge/sparc-service/sparc_service/models.py b/authbridge/sparc-service/sparc_service/models.py new file mode 100644 index 000000000..60ed20678 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/models.py @@ -0,0 +1,60 @@ +"""Wire models for the SPARC reflection service. + +The request mirrors SPARC's native input — an OpenAI-style conversation, the +available tool inventory, and the proposed tool call(s). The response is a thin, +faithful projection of SPARC's verdict; enforcement policy is the caller's job. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + + +class ReflectRequest(BaseModel): + """Inputs SPARC needs to judge a proposed tool call.""" + + messages: list[dict[str, Any]] = Field( + default_factory=list, + description="OpenAI-style conversation context (role/content turns).", + ) + tool_specs: list[dict[str, Any]] = Field( + default_factory=list, + description="Available tools in OpenAI function-calling format.", + ) + tool_calls: list[dict[str, Any]] = Field( + ..., + min_length=1, + description="Proposed tool call(s); SPARC evaluates the first.", + ) + session_id: str | None = Field(default=None, description="Opaque correlation id for logs/observability.") + track: str | None = Field( + default=None, + description="Per-request SPARC track override; falls back to the service's SPARC_TRACK.", + ) + + +class ReflectionIssue(BaseModel): + """A single grounding/appropriateness problem SPARC found.""" + + issue_type: str | None = None + metric_name: str | None = None + explanation: str | None = None + correction: Any | None = None + + +class ReflectResponse(BaseModel): + """SPARC's verdict, faithfully projected for the AuthBridge plugin.""" + + decision: str = Field(description="approve | reject | error") + issues: list[ReflectionIssue] = Field(default_factory=list) + overall_avg_score: float | None = Field( + default=None, + description="SPARC's grounding score, normalized 0..1 (higher = better grounded), when available.", + ) + execution_time_ms: float | None = None + raw_pipeline_result: dict[str, Any] | None = Field( + default=None, + description="Full SPARC pipeline detail; included only when SPARC_INCLUDE_RAW_RESPONSE is set.", + ) diff --git a/authbridge/sparc-service/sparc_service/providers.py b/authbridge/sparc-service/sparc_service/providers.py new file mode 100644 index 000000000..6489926b1 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/providers.py @@ -0,0 +1,116 @@ +"""ALTK LLM-client and SPARC-component construction. + +Isolates every import of the heavy ``altk`` package so the rest of the service +(settings, models, API wiring) stays import-light and unit-testable without +network access or LLM credentials. +""" + +from __future__ import annotations + +import os +from typing import Any + +from .settings import Settings + +# Map the configurable track names to altk Track enum members. Resolved lazily. +_TRACK_NAMES = { + "fast_track": "FAST_TRACK", + "slow_track": "SLOW_TRACK", + "syntax": "SYNTAX", + "spec_free": "SPEC_FREE", + "transformations_only": "TRANSFORMATIONS_ONLY", +} + + +def resolve_track(name: str): + """Return the altk ``Track`` enum member for a configured track name.""" + from altk.pre_tool.core import Track + + return getattr(Track, _TRACK_NAMES[name]) + + +def build_llm_client(settings: Settings): + """Construct a validating ALTK LLM client for the configured provider. + + watsonx (default) and ollama use ALTK's provider-native LiteLLM validating + clients. Every other provider — openai, azure (Azure OpenAI), and the + generic ``litellm`` escape hatch — uses ALTK's generic LiteLLM validating + client (``litellm.output_val``), where the model string selects the provider + and any extra client kwargs come from ``SPARC_LLM_KWARGS_JSON``. SPARC + requires a *validating* client, so every branch returns one. + """ + from altk.core.llm import get_llm + + client_cls = get_llm(settings.registry_id) + + # An explicit SPARC_LLM_REGISTRY_ID override means the caller picked the ALTK + # client class directly, so its constructor may not accept the provider-native + # kwargs below (e.g. watsonx's project_id). In that case skip the native + # branches and use the generic LiteLLM-kwargs path, where everything the + # client needs comes from SPARC_LLM_KWARGS_JSON. + native = not settings.llm_registry_id + + if native and settings.provider == "watsonx": + return client_cls( + model_name=settings.model, + api_key=settings.wx_api_key, + project_id=settings.wx_project_id, + api_base=settings.wx_url, + timeout=settings.llm_timeout_seconds, + ) + + if native and settings.provider == "ollama": + # Point every LiteLLM Ollama call at the configured server. We pass + # api_url EXPLICITLY (the ALTK ollama client's param) so ALTK's internal + # metric sub-clients inherit it — relying on the OLLAMA_API_BASE env var + # alone is not enough (those sub-calls otherwise fall back to + # localhost:11434). The env vars are set too as a belt-and-suspenders. + os.environ["OLLAMA_API_BASE"] = settings.ollama_base_url + os.environ["OLLAMA_BASE_URL"] = settings.ollama_base_url + return client_cls( + model_name=settings.model, + api_key=settings.ollama_api_key, + api_url=settings.ollama_base_url, + ) + + # Generic path (openai / azure / litellm, or any provider when + # SPARC_LLM_REGISTRY_ID is set). LiteLLM routes by the model string and reads + # provider API keys from the environment (OPENAI_API_KEY, AZURE_API_KEY, + # ANTHROPIC_API_KEY, ...) when not supplied; SPARC_LLM_KWARGS_JSON supplies + # anything else (api_base, api_version, api_key, deployment, ...). When a + # registry override selects a non-LiteLLM client, supply its constructor + # kwargs via SPARC_LLM_KWARGS_JSON. + lite_kwargs: dict[str, Any] = dict(settings.llm_kwargs) + lite_kwargs.setdefault("timeout", settings.llm_timeout_seconds) + if settings.provider == "openai" and settings.openai_base_url and "api_base" not in lite_kwargs: + lite_kwargs["api_base"] = settings.openai_base_url + return client_cls(model_name=settings.model, **lite_kwargs) + + +def build_component(settings: Settings): + """Construct the SPARC reflection component from settings. + + This performs provider authentication eagerly for watsonx/openai (the + validating clients authenticate on construction), so callers should treat a + raised exception as "not ready". + """ + # Mirror the cost-map flag the reference reflector sets for LiteLLM. + os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + from altk.core.toolkit import ComponentConfig + from altk.pre_tool.core import SPARCExecutionMode + from altk.pre_tool.sparc import SPARCReflectionComponent + + config = ComponentConfig(llm_client=build_llm_client(settings)) + component = SPARCReflectionComponent( + config=config, + track=resolve_track(settings.track), + execution_mode=SPARCExecutionMode.ASYNC, + include_raw_response=settings.include_raw_response, + retries=settings.retries, + max_parallel=settings.max_parallel, + ) + init_error = getattr(component, "_initialization_error", None) + if init_error: + raise RuntimeError(f"SPARC component failed to initialize: {init_error}") + return component diff --git a/authbridge/sparc-service/sparc_service/settings.py b/authbridge/sparc-service/sparc_service/settings.py new file mode 100644 index 000000000..12023b1c4 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/settings.py @@ -0,0 +1,175 @@ +"""Environment-driven configuration for the SPARC reflection service. + +Configuration is intentionally read from the environment (12-factor) so the +service can be tuned via a Kubernetes ConfigMap/Secret without rebuilding the +image. Watsonx credentials follow the same ``WX_*`` (with ``WATSONX_*`` +fallbacks) convention used elsewhere in Kagenti. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field + +# Supported SPARC LLM providers and the ALTK validating-client registry id each +# maps to. All entries are *validating* clients, which SPARC requires. +# +# watsonx (default) and ollama use ALTK's provider-native LiteLLM clients. +# Everything else — openai, azure (Azure OpenAI), and the generic "litellm" +# escape hatch — uses ALTK's generic LiteLLM client (litellm.output_val), which +# routes by the model string (e.g. "gpt-4o-mini", "azure/", +# "anthropic/claude-3-5-sonnet", "gemini/gemini-1.5-pro", "bedrock/..."). This +# means ANY provider LiteLLM/ALTK supports works via config alone — set +# SPARC_LLM_PROVIDER=litellm, SPARC_MODEL=, and pass any extra +# client kwargs (api_base, api_version, api_key, ...) via SPARC_LLM_KWARGS_JSON. +# For full control you can also set SPARC_LLM_REGISTRY_ID to any ALTK registry id. +PROVIDER_REGISTRY_IDS: dict[str, str] = { + "watsonx": "litellm.watsonx.output_val", + "ollama": "litellm.ollama.output_val", + "openai": "litellm.output_val", + "azure": "litellm.output_val", + "litellm": "litellm.output_val", +} + +# Per-provider default model id (empty → SPARC_MODEL is required). +PROVIDER_DEFAULT_MODELS: dict[str, str] = { + "watsonx": "mistral-large-2512", + "ollama": "llama3.2:3b", + "openai": "gpt-4o-mini", + "azure": "", + "litellm": "", +} + +# SPARC reflection tracks exposed via configuration. Mapped to altk Track enum +# lazily in providers.py to avoid importing heavy altk modules at settings time. +SUPPORTED_TRACKS = {"fast_track", "slow_track", "syntax", "spec_free", "transformations_only"} + + +def _truthy(value: str) -> bool: + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _int_env(name: str, default: int) -> int: + raw = os.getenv(name, "").strip() + if not raw: + return default + try: + return int(raw) + except ValueError: + return default + + +@dataclass(frozen=True) +class Settings: + """Immutable, validated view of the service configuration.""" + + provider: str = "watsonx" + model: str = "mistral-large-2512" + track: str = "fast_track" + + # SPARC execution tuning. + llm_timeout_seconds: int = 120 + retries: int = 3 + max_parallel: int = 2 + include_raw_response: bool = True + + # Watsonx credentials (provider == "watsonx"). + wx_api_key: str = "" + wx_project_id: str = "" + wx_url: str = "https://us-south.ml.cloud.ibm.com" + + # Ollama (provider == "ollama"). + ollama_base_url: str = "http://localhost:11434" + ollama_api_key: str = "ollama" + + # OpenAI (provider == "openai"). + openai_api_key: str = "" + openai_base_url: str = "" + + # Generic LiteLLM client kwargs (provider in {openai, azure, litellm}), e.g. + # {"api_base": "...", "api_version": "...", "api_key": "..."}. Parsed from + # SPARC_LLM_KWARGS_JSON. Passed straight to the ALTK litellm client. + llm_kwargs: dict = field(default_factory=dict) + # Optional explicit ALTK registry id override (advanced; any ALTK client). + llm_registry_id: str = "" + + # HTTP server. + host: str = "0.0.0.0" + port: int = 8090 + + # Validation errors collected at load time (provider creds missing, etc.). + errors: tuple[str, ...] = field(default_factory=tuple) + + @property + def registry_id(self) -> str: + return self.llm_registry_id or PROVIDER_REGISTRY_IDS[self.provider] + + def credentials_present(self) -> bool: + return not self.errors + + @classmethod + def from_env(cls) -> "Settings": + provider = os.getenv("SPARC_LLM_PROVIDER", "watsonx").strip().lower() + errors: list[str] = [] + if provider not in PROVIDER_REGISTRY_IDS: + errors.append( + f"unsupported SPARC_LLM_PROVIDER={provider!r}; expected one of {sorted(PROVIDER_REGISTRY_IDS)}" + ) + provider = "watsonx" + + track = os.getenv("SPARC_TRACK", "fast_track").strip().lower() + if track not in SUPPORTED_TRACKS: + errors.append(f"unsupported SPARC_TRACK={track!r}; expected one of {sorted(SUPPORTED_TRACKS)}") + track = "fast_track" + + wx_api_key = os.getenv("WX_API_KEY") or os.getenv("WATSONX_API_KEY") or "" + wx_project_id = os.getenv("WX_PROJECT_ID") or os.getenv("WATSONX_PROJECT_ID") or "" + wx_url = os.getenv("WX_URL") or os.getenv("WATSONX_URL") or "https://us-south.ml.cloud.ibm.com" + + model = os.getenv("SPARC_MODEL") or os.getenv("WX_MODEL_ID") or PROVIDER_DEFAULT_MODELS.get(provider, "") + + # Optional generic LiteLLM client kwargs (azure/openai/litellm). + llm_kwargs: dict = {} + raw_kwargs = os.getenv("SPARC_LLM_KWARGS_JSON", "").strip() + if raw_kwargs: + try: + parsed = json.loads(raw_kwargs) + if isinstance(parsed, dict): + llm_kwargs = parsed + else: + errors.append("SPARC_LLM_KWARGS_JSON must be a JSON object") + except ValueError as exc: + errors.append(f"SPARC_LLM_KWARGS_JSON is not valid JSON: {exc}") + + # Provider-specific credential / config validation. + if provider == "watsonx" and (not wx_api_key or not wx_project_id): + errors.append("provider=watsonx requires WX_API_KEY and WX_PROJECT_ID") + if provider == "openai" and not (os.getenv("OPENAI_API_KEY") or "api_key" in llm_kwargs): + errors.append("provider=openai requires OPENAI_API_KEY (or api_key in SPARC_LLM_KWARGS_JSON)") + if provider in ("azure", "litellm") and not model: + errors.append( + f"provider={provider} requires SPARC_MODEL (e.g. azure/ or anthropic/claude-3-5-sonnet)" + ) + + return cls( + provider=provider, + model=model, + track=track, + llm_timeout_seconds=_int_env("SPARC_LLM_TIMEOUT", 120), + retries=_int_env("SPARC_RETRIES", 3), + max_parallel=_int_env("SPARC_MAX_PARALLEL", 2), + include_raw_response=_truthy(os.getenv("SPARC_INCLUDE_RAW_RESPONSE", "true")), + wx_api_key=wx_api_key, + wx_project_id=wx_project_id, + wx_url=wx_url, + ollama_base_url=os.getenv("OLLAMA_BASE_URL", "http://localhost:11434"), + ollama_api_key=os.getenv("OLLAMA_API_KEY", "ollama"), + openai_api_key=os.getenv("OPENAI_API_KEY", ""), + openai_base_url=os.getenv("OPENAI_BASE_URL", ""), + llm_kwargs=llm_kwargs, + llm_registry_id=os.getenv("SPARC_LLM_REGISTRY_ID", "").strip(), + host=os.getenv("HOST", "0.0.0.0"), + port=_int_env("PORT", 8090), + errors=tuple(errors), + ) diff --git a/authbridge/sparc-service/tests/conftest.py b/authbridge/sparc-service/tests/conftest.py new file mode 100644 index 000000000..59d6e1d23 --- /dev/null +++ b/authbridge/sparc-service/tests/conftest.py @@ -0,0 +1,68 @@ +"""Shared test fixtures. + +The unit tests never touch the network or LLM credentials: they inject a fake +SPARC component (built from real altk result types) into the engine. +""" + +from __future__ import annotations + +import os +from types import SimpleNamespace +from typing import Any + +import pytest +from sparc_service.engine import ReflectionEngine +from sparc_service.settings import Settings + + +def make_reflection_output( + decision: str, + issues: list[dict[str, Any]] | None = None, + overall_avg_score: float | None = None, + execution_time_ms: float = 12.3, +) -> Any: + """Build a real altk reflection output object for fakes.""" + from altk.pre_tool.core import ( + SPARCReflectionDecision, + SPARCReflectionIssue, + SPARCReflectionResult, + ) + + issue_objs = [SPARCReflectionIssue(**i) for i in (issues or [])] + reflection = SPARCReflectionResult(decision=SPARCReflectionDecision(decision), issues=issue_objs) + raw_pipeline = {} + if overall_avg_score is not None: + raw_pipeline["overall_avg_score"] = overall_avg_score + return SimpleNamespace( + output=SimpleNamespace( + reflection_result=reflection, + execution_time_ms=execution_time_ms, + raw_pipeline_result=raw_pipeline, + ) + ) + + +class FakeComponent: + """Stands in for SPARCReflectionComponent; returns a canned verdict.""" + + def __init__(self, output: Any) -> None: + self._output = output + self.calls: list[Any] = [] + + def process(self, run_input: Any, phase: Any) -> Any: + self.calls.append(run_input) + return self._output + + +def make_engine(output: Any, settings: Settings | None = None) -> ReflectionEngine: + settings = settings or Settings(provider="watsonx", wx_api_key="k", wx_project_id="p") + component = FakeComponent(output) + return ReflectionEngine(settings, component_factory=lambda _track: component) + + +@pytest.fixture +def clean_env(monkeypatch): + for key in list(os.environ): + if key.startswith(("SPARC_", "WX_", "WATSONX_", "OLLAMA_", "OPENAI_")): + monkeypatch.delenv(key, raising=False) + yield monkeypatch diff --git a/authbridge/sparc-service/tests/test_api.py b/authbridge/sparc-service/tests/test_api.py new file mode 100644 index 000000000..f33a3598b --- /dev/null +++ b/authbridge/sparc-service/tests/test_api.py @@ -0,0 +1,110 @@ +"""HTTP surface tests via FastAPI TestClient with an injected fake engine.""" + +from __future__ import annotations + +import json + +from fastapi.testclient import TestClient +from sparc_service.api import create_app +from sparc_service.settings import Settings + +from tests.conftest import make_engine, make_reflection_output + +REQUEST_BODY = { + "messages": [{"role": "user", "content": "Refund transaction TX482."}], + "tool_specs": [ + { + "type": "function", + "function": { + "name": "get_transaction", + "parameters": { + "type": "object", + "properties": {"transaction_id": {"type": "string"}}, + "required": ["transaction_id"], + }, + }, + } + ], + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "get_transaction", "arguments": json.dumps({"transaction_id": "TX4821"})}, + } + ], + "session_id": "s1", +} + + +def _client(output) -> TestClient: + return TestClient(create_app(engine=make_engine(output))) + + +def test_reflect_reject(): + output = make_reflection_output( + "reject", + issues=[ + {"issue_type": "semantic_function", "metric_name": "m", "explanation": "no refund tool", "correction": None} + ], + overall_avg_score=2.0, + ) + resp = _client(output).post("/reflect", json=REQUEST_BODY) + assert resp.status_code == 200 + body = resp.json() + assert body["decision"] == "reject" + assert body["overall_avg_score"] == 2.0 + assert body["issues"][0]["explanation"] == "no refund tool" + + +def test_reflect_requires_tool_calls(): + bad = dict(REQUEST_BODY, tool_calls=[]) + resp = _client(make_reflection_output("approve")).post("/reflect", json=bad) + assert resp.status_code == 422 # pydantic validation: min_length=1 + + +def test_healthz_reports_provider(): + resp = _client(make_reflection_output("approve")).get("/healthz") + assert resp.status_code == 200 + assert resp.json()["provider"] == "watsonx" + + +def test_readyz_ok_with_buildable_component(): + resp = _client(make_reflection_output("approve")).get("/readyz") + assert resp.status_code == 200 + assert resp.json()["status"] == "ready" + + +def test_readyz_503_when_config_invalid(): + # Settings with a recorded error -> not ready, never builds component. + settings = Settings(provider="watsonx", errors=("provider=watsonx requires WX_API_KEY and WX_PROJECT_ID",)) + engine = make_engine(make_reflection_output("approve"), settings) + client = TestClient(create_app(engine=engine)) + resp = client.get("/readyz") + assert resp.status_code == 503 + + +def test_reflect_502_hides_backend_error(): + # Provider exception text can embed endpoints/credentials, so /reflect must + # never echo it — it returns a stable message and logs the detail instead. + class BoomEngine: + settings = Settings(provider="watsonx", wx_api_key="k", wx_project_id="p") + + def reflect(self, request): + raise RuntimeError("watsonx exploded with key sk-secret-123") + + def ready(self): + return True, None + + client = TestClient(create_app(engine=BoomEngine())) + resp = client.post("/reflect", json=REQUEST_BODY) + assert resp.status_code == 502 + body = json.dumps(resp.json()) + assert "sk-secret-123" not in body + assert "watsonx exploded" not in body + assert resp.json()["detail"]["error"] == "reflection failed" + + +def test_reflect_400_on_unsupported_track(): + bad = dict(REQUEST_BODY, track="not_a_track") + resp = _client(make_reflection_output("approve")).post("/reflect", json=bad) + assert resp.status_code == 400 diff --git a/authbridge/sparc-service/tests/test_engine.py b/authbridge/sparc-service/tests/test_engine.py new file mode 100644 index 000000000..c0a61de7c --- /dev/null +++ b/authbridge/sparc-service/tests/test_engine.py @@ -0,0 +1,115 @@ +"""Engine IO mapping, using a fake SPARC component (no network).""" + +from __future__ import annotations + +import json + +from sparc_service.models import ReflectRequest + +from tests.conftest import make_engine, make_reflection_output + +FINANCE_TOOL_SPECS = [ + { + "type": "function", + "function": { + "name": "get_transaction", + "description": "Fetch transaction details by transaction id.", + "parameters": { + "type": "object", + "properties": {"transaction_id": {"type": "string"}}, + "required": ["transaction_id"], + }, + }, + } +] + + +def _request(args: dict) -> ReflectRequest: + return ReflectRequest( + messages=[{"role": "user", "content": "Refund transaction TX482."}], + tool_specs=FINANCE_TOOL_SPECS, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_transaction", "arguments": json.dumps(args)}, + } + ], + session_id="s1", + ) + + +def test_approve_maps_cleanly(): + engine = make_engine(make_reflection_output("approve", overall_avg_score=4.5)) + resp = engine.reflect(_request({"transaction_id": "TX4827"})) + assert resp.decision == "approve" + assert resp.issues == [] + assert resp.overall_avg_score == 4.5 + assert resp.execution_time_ms == 12.3 + + +def test_reject_carries_issue_and_correction(): + output = make_reflection_output( + "reject", + issues=[ + { + "issue_type": "semantic_function", + "metric_name": "function_selection_appropriateness", + "explanation": "get_transaction does not process refunds.", + "correction": {"corrected_function_name": "no_function"}, + } + ], + overall_avg_score=2.0, + ) + engine = make_engine(output) + resp = engine.reflect(_request({"transaction_id": "TX4821"})) + assert resp.decision == "reject" + assert len(resp.issues) == 1 + issue = resp.issues[0] + assert issue.metric_name == "function_selection_appropriateness" + assert "refund" in issue.explanation + assert issue.correction["corrected_function_name"] == "no_function" + assert resp.overall_avg_score == 2.0 + + +def test_error_decision_passthrough(): + engine = make_engine(make_reflection_output("error")) + resp = engine.reflect(_request({"transaction_id": "TX4821"})) + assert resp.decision == "error" + + +def test_raw_pipeline_suppressed_when_disabled(): + from sparc_service.settings import Settings + + settings = Settings(provider="watsonx", wx_api_key="k", wx_project_id="p", include_raw_response=False) + engine = make_engine(make_reflection_output("approve", overall_avg_score=5.0), settings) + resp = engine.reflect(_request({"transaction_id": "TX4827"})) + # Score is still surfaced, but the verbose raw pipeline is withheld. + assert resp.overall_avg_score == 5.0 + assert resp.raw_pipeline_result is None + + +def test_run_input_receives_request_fields(): + engine = make_engine(make_reflection_output("approve")) + engine.reflect(_request({"transaction_id": "TX4827"})) + # The fake records the run_input the engine handed to SPARC. + component = next(iter(engine._components.values())) # noqa: SLF001 - test introspection + assert len(component.calls) == 1 + run_input = component.calls[0] + # All three SPARC inputs reach the component unchanged. + assert run_input.messages == [{"role": "user", "content": "Refund transaction TX482."}] + assert run_input.tool_specs == FINANCE_TOOL_SPECS + assert run_input.tool_calls[0]["function"]["name"] == "get_transaction" + assert run_input.tool_calls[0]["function"]["arguments"] == json.dumps({"transaction_id": "TX4827"}) + + +def test_unsupported_track_rejected(): + engine = make_engine(make_reflection_output("approve")) + request = _request({"transaction_id": "TX4827"}) + request.track = "made_up_track" + try: + engine.reflect(request) + except ValueError as exc: + assert "unsupported track" in str(exc) + else: + raise AssertionError("expected ValueError for an unsupported track") diff --git a/authbridge/sparc-service/tests/test_providers.py b/authbridge/sparc-service/tests/test_providers.py new file mode 100644 index 000000000..89199a64d --- /dev/null +++ b/authbridge/sparc-service/tests/test_providers.py @@ -0,0 +1,52 @@ +"""LLM-client construction: provider-native vs explicit registry override. + +These tests stub ALTK's ``get_llm`` so they exercise the kwarg-selection logic +in ``build_llm_client`` without authenticating against any real provider. +""" + +from __future__ import annotations + +from typing import Any + +import altk.core.llm as llm_mod +from sparc_service.providers import build_llm_client +from sparc_service.settings import Settings + + +def _capture(monkeypatch) -> dict[str, Any]: + captured: dict[str, Any] = {} + + class FakeClient: + def __init__(self, **kwargs: Any) -> None: + captured.update(kwargs) + + monkeypatch.setattr(llm_mod, "get_llm", lambda _registry_id: FakeClient) + return captured + + +def test_native_watsonx_passes_provider_kwargs(monkeypatch): + captured = _capture(monkeypatch) + s = Settings(provider="watsonx", wx_api_key="k", wx_project_id="p", model="mistral-large-2512") + build_llm_client(s) + assert captured["project_id"] == "p" + assert captured["model_name"] == "mistral-large-2512" + + +def test_registry_override_skips_provider_kwargs(monkeypatch): + # With an explicit registry override, the watsonx-native project_id/api_base + # kwargs must NOT be passed (they'd break a generic LiteLLM client). Only the + # generic kwargs from llm_kwargs reach the constructor. + captured = _capture(monkeypatch) + s = Settings( + provider="watsonx", + wx_api_key="k", + wx_project_id="p", + model="gpt-4o-mini", + llm_registry_id="litellm.output_val", + llm_kwargs={"api_key": "sk-x"}, + ) + build_llm_client(s) + assert "project_id" not in captured + assert "api_base" not in captured + assert captured["model_name"] == "gpt-4o-mini" + assert captured["api_key"] == "sk-x" diff --git a/authbridge/sparc-service/tests/test_settings.py b/authbridge/sparc-service/tests/test_settings.py new file mode 100644 index 000000000..0d2ef0dd3 --- /dev/null +++ b/authbridge/sparc-service/tests/test_settings.py @@ -0,0 +1,105 @@ +"""Settings parsing and provider-credential validation.""" + +from __future__ import annotations + +from sparc_service.settings import Settings + + +def test_defaults_to_watsonx(clean_env): + clean_env.setenv("WX_API_KEY", "key") + clean_env.setenv("WX_PROJECT_ID", "proj") + s = Settings.from_env() + assert s.provider == "watsonx" + assert s.registry_id == "litellm.watsonx.output_val" + assert s.model == "mistral-large-2512" + assert s.credentials_present() + + +def test_watsonx_missing_creds_records_error(clean_env): + s = Settings.from_env() + assert not s.credentials_present() + assert any("WX_API_KEY" in e for e in s.errors) + + +def test_watsonx_fallback_env_names(clean_env): + clean_env.setenv("WATSONX_API_KEY", "key") + clean_env.setenv("WATSONX_PROJECT_ID", "proj") + s = Settings.from_env() + assert s.wx_api_key == "key" + assert s.wx_project_id == "proj" + assert s.credentials_present() + + +def test_ollama_provider_needs_no_watsonx(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "ollama") + s = Settings.from_env() + assert s.provider == "ollama" + assert s.registry_id == "litellm.ollama.output_val" + assert s.model == "llama3.2:3b" + assert s.credentials_present() + + +def test_openai_provider_requires_key(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "openai") + s = Settings.from_env() + assert not s.credentials_present() + clean_env.setenv("OPENAI_API_KEY", "sk-x") + assert Settings.from_env().credentials_present() + + +def test_unsupported_provider_records_error(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "bogus") + s = Settings.from_env() + assert any("unsupported SPARC_LLM_PROVIDER" in e for e in s.errors) + + +def test_model_override(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "ollama") + clean_env.setenv("SPARC_MODEL", "qwen2.5:7b") + assert Settings.from_env().model == "qwen2.5:7b" + + +def test_generic_litellm_provider(clean_env): + # The generic escape hatch supports any LiteLLM model (anthropic/gemini/...). + clean_env.setenv("SPARC_LLM_PROVIDER", "litellm") + clean_env.setenv("SPARC_MODEL", "anthropic/claude-3-5-sonnet") + clean_env.setenv("SPARC_LLM_KWARGS_JSON", '{"api_key": "sk-ant-x"}') + s = Settings.from_env() + assert s.registry_id == "litellm.output_val" + assert s.model == "anthropic/claude-3-5-sonnet" + assert s.llm_kwargs == {"api_key": "sk-ant-x"} + assert s.credentials_present() + + +def test_litellm_provider_requires_model(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "litellm") + s = Settings.from_env() + assert any("requires SPARC_MODEL" in e for e in s.errors) + + +def test_azure_provider(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "azure") + clean_env.setenv("SPARC_MODEL", "azure/my-deployment") + clean_env.setenv( + "SPARC_LLM_KWARGS_JSON", + '{"api_base": "https://x.openai.azure.com", "api_version": "2024-06-01", "api_key": "k"}', + ) + s = Settings.from_env() + assert s.registry_id == "litellm.output_val" + assert s.llm_kwargs["api_version"] == "2024-06-01" + assert s.credentials_present() + + +def test_invalid_kwargs_json_records_error(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "litellm") + clean_env.setenv("SPARC_MODEL", "anthropic/claude-3-5-sonnet") + clean_env.setenv("SPARC_LLM_KWARGS_JSON", "{not json}") + s = Settings.from_env() + assert any("SPARC_LLM_KWARGS_JSON" in e for e in s.errors) + + +def test_registry_id_override(clean_env): + clean_env.setenv("WX_API_KEY", "key") + clean_env.setenv("WX_PROJECT_ID", "proj") + clean_env.setenv("SPARC_LLM_REGISTRY_ID", "azure_openai.sync.output_val") + assert Settings.from_env().registry_id == "azure_openai.sync.output_val" diff --git a/authbridge/sparc-service/tests/test_watsonx_integration.py b/authbridge/sparc-service/tests/test_watsonx_integration.py new file mode 100644 index 000000000..cd20224d4 --- /dev/null +++ b/authbridge/sparc-service/tests/test_watsonx_integration.py @@ -0,0 +1,86 @@ +"""Opt-in integration test against real watsonx. + +Skipped unless RUN_WATSONX_TESTS=1 and WX_* credentials are present. Validates +the canonical finance scenario end-to-end through the real SPARC engine: +an ungrounded/inappropriate tool call must be rejected. + +Run with: + RUN_WATSONX_TESTS=1 pytest tests/test_watsonx_integration.py -v +""" + +from __future__ import annotations + +import json +import os + +import pytest +from sparc_service.engine import ReflectionEngine +from sparc_service.models import ReflectRequest +from sparc_service.settings import Settings + +pytestmark = pytest.mark.skipif( + os.getenv("RUN_WATSONX_TESTS") != "1" or not (os.getenv("WX_API_KEY") or os.getenv("WATSONX_API_KEY")), + reason="set RUN_WATSONX_TESTS=1 and WX_* creds to run watsonx integration tests", +) + +TOOL_SPECS = [ + { + "type": "function", + "function": { + "name": "get_transaction", + "description": "Fetch transaction details by transaction id.", + "parameters": { + "type": "object", + "properties": {"transaction_id": {"type": "string", "description": "Exact transaction identifier."}}, + "required": ["transaction_id"], + }, + }, + } +] + + +def _engine() -> ReflectionEngine: + return ReflectionEngine(Settings.from_env()) + + +def test_ungrounded_partial_id_is_rejected(): + """User gives partial 'TX482'; model hallucinates 'TX4821' -> reject.""" + req = ReflectRequest( + messages=[ + {"role": "user", "content": "Refund transaction TX482 because it was a duplicate charge."}, + {"role": "assistant", "content": "I will process the refund."}, + ], + tool_specs=TOOL_SPECS, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_transaction", "arguments": json.dumps({"transaction_id": "TX4821"})}, + } + ], + session_id="itest-reject", + ) + resp = _engine().reflect(req) + assert resp.decision in {"reject", "error"} + assert resp.decision == "reject", f"expected reject, got {resp.decision}: {resp.issues}" + assert resp.issues, "reject should carry at least one issue" + + +def test_grounded_full_id_is_approved(): + """Once the user provides the exact id, the same call is grounded -> approve.""" + req = ReflectRequest( + messages=[ + {"role": "user", "content": "Look up transaction TX4827 for me."}, + ], + tool_specs=TOOL_SPECS, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_transaction", "arguments": json.dumps({"transaction_id": "TX4827"})}, + } + ], + session_id="itest-approve", + ) + resp = _engine().reflect(req) + assert resp.decision == "approve", f"expected approve, got {resp.decision}: {resp.issues}"