diff --git a/authbridge/CLAUDE.md b/authbridge/CLAUDE.md index cb67b20e8..ad52f7f5f 100644 --- a/authbridge/CLAUDE.md +++ b/authbridge/CLAUDE.md @@ -11,7 +11,7 @@ binaries with shared auth logic in `authlib/`: - `cmd/authbridge-proxy/` — proxy-sidecar mode (default). HTTP forward + reverse proxies. Full plugin set (jwt-validation, token-exchange, a2a-parser, - mcp-parser, inference-parser, ibac). + mcp-parser, inference-parser, ibac, sparc). - `cmd/authbridge-envoy/` — envoy-sidecar mode. ext_proc gRPC server hooked into Envoy. Full plugin set. - `cmd/authbridge-lite/` — proxy-sidecar mode, lite plugin set (auth gates diff --git a/authbridge/authlib/plugins/sparc/collect.go b/authbridge/authlib/plugins/sparc/collect.go new file mode 100644 index 000000000..5c66c29de --- /dev/null +++ b/authbridge/authlib/plugins/sparc/collect.go @@ -0,0 +1,165 @@ +package sparc + +import ( + "encoding/json" + "fmt" + "path" + "strings" + + "github.com/kagenti/kagenti-extensions/authbridge/authlib/pipeline" +) + +// correlateInferenceContext pulls the conversation (incl. system prompt) and +// the tool inventory from the session's most recent outbound inference request +// — i.e. exactly what the agent sent to its LLM. Used in mcp mode, where the +// concrete tool call comes from the MCP request but the context must be sourced +// from the agent's LLM call. Returns ok=false when no inference event exists +// (no inference-parser, or session tracking off) so the caller can skip rather +// than reflect on partial data. +func correlateInferenceContext(pctx *pipeline.Context) (messages, toolSpecs []map[string]any, ok bool) { + if pctx.Session == nil { + return nil, nil, false + } + infs := pctx.Session.InferenceRequests() + if len(infs) == 0 { + return nil, nil, false + } + inf := infs[len(infs)-1].Inference + if inf == nil || len(inf.Messages) == 0 { + return nil, nil, false + } + messages, toolSpecs = inferenceMessagesAndTools(inf) + return messages, toolSpecs, true +} + +// inferenceInputs extracts messages + tool specs directly from the inference +// extension on the current request/response (inference mode — all inputs are +// co-located, no correlation needed). +func inferenceInputs(inf *pipeline.InferenceExtension) (messages, toolSpecs []map[string]any, ok bool) { + if inf == nil || len(inf.Messages) == 0 { + return nil, nil, false + } + messages, toolSpecs = inferenceMessagesAndTools(inf) + return messages, toolSpecs, true +} + +func inferenceMessagesAndTools(inf *pipeline.InferenceExtension) (messages, toolSpecs []map[string]any) { + for _, m := range inf.Messages { + messages = append(messages, map[string]any{"role": m.Role, "content": m.Content}) + } + for _, t := range inf.Tools { + toolSpecs = append(toolSpecs, map[string]any{ + "type": "function", + "function": map[string]any{ + "name": t.Name, + "description": t.Description, + "parameters": t.Parameters, + }, + }) + } + return messages, toolSpecs +} + +// buildToolCall renders the MCP tools/call as an OpenAI-style tool call (the +// shape SPARC expects). Arguments are serialized to a JSON string. +func buildToolCall(mcp *pipeline.MCPExtension, toolName string) map[string]any { + return openAIToolCall(fmt.Sprintf("%v", mcpRPCID(mcp)), toolName, extractMCPToolArgs(mcp)) +} + +// openAIToolCall builds the canonical OpenAI function tool-call object. args is +// a JSON string (defaulted to "{}"). +func openAIToolCall(id, name, args string) map[string]any { + if args == "" { + args = "{}" + } + return map[string]any{ + "id": id, + "type": "function", + "function": map[string]any{"name": name, "arguments": args}, + } +} + +func extractMCPToolName(mcp *pipeline.MCPExtension) string { + if mcp == nil || mcp.Method != "tools/call" { + return "" + } + if name, ok := mcp.Params["name"].(string); ok { + return name + } + return "" +} + +func extractMCPToolArgs(mcp *pipeline.MCPExtension) string { + if mcp == nil || mcp.Method != "tools/call" { + return "" + } + args, ok := mcp.Params["arguments"] + if !ok { + return "" + } + b, err := json.Marshal(args) + if err != nil { + return "" + } + return string(b) +} + +func mcpRPCID(mcp *pipeline.MCPExtension) any { + if mcp == nil { + return nil + } + return mcp.RPCID +} + +func sessionID(pctx *pipeline.Context) string { + if pctx.Session != nil { + return pctx.Session.ID + } + return "" +} + +func firstExplanation(verdict ReflectVerdict) string { + for _, iss := range verdict.Issues { + if iss.Explanation != "" { + return iss.Explanation + } + } + return "tool call not grounded in the conversation" +} + +func scoreString(score *float64) string { + if score == nil { + return "" + } + return fmt.Sprintf("%.2f", *score) +} + +// anyGlobMatch reports whether name matches any path.Match glob in patterns. +func anyGlobMatch(patterns []string, name string) bool { + for _, p := range patterns { + if matched, _ := path.Match(p, name); matched { + return true + } + } + return false +} + +// matchesAnyHost reports whether host matches any glob (path.Match), port stripped. +func matchesAnyHost(patterns []string, host string) bool { + if host == "" { + return false + } + if i := strings.IndexByte(host, ':'); i >= 0 { + host = host[:i] + } + return anyGlobMatch(patterns, host) +} + +// preview truncates s to n runes, appending an ellipsis when truncated. +func preview(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} diff --git a/authbridge/authlib/plugins/sparc/plugin.go b/authbridge/authlib/plugins/sparc/plugin.go new file mode 100644 index 000000000..012ded3f9 --- /dev/null +++ b/authbridge/authlib/plugins/sparc/plugin.go @@ -0,0 +1,513 @@ +// Package sparc implements a pre-tool reflection plugin for authbridge. +// +// Before an agent's proposed tool call executes, the plugin asks an external +// SPARC reflection service (the agent-lifecycle-toolkit / ALTK SPARC component, +// served over HTTP) whether the call is *grounded* in the conversation and the +// available tool specifications. SPARC catches hallucinated / ungrounded +// arguments (e.g. an invented transaction id) and inappropriate tool selection. +// +// Generic by design — works for ANY kagenti agent. SPARC's three inputs are +// collected from exactly what the agent produces, with no bespoke per-agent +// wiring: +// +// - conversation history (INCLUDING the system prompt) — from the agent's +// LLM call captured by inference-parser (pctx.Extensions.Inference.Messages, +// which preserves every role verbatim); +// - tool specifications in OpenAI function-calling format — from the same +// inference call (pctx.Extensions.Inference.Tools); +// - the proposed tool call in OpenAI format — from the outbound MCP +// tools/call (mcp mode) or from the LLM response (inference mode). +// +// Enforcement is format-aware (the verdict is returned to the agent in the +// shape it expects): +// +// - enforcement: "mcp" (default; the kagenti norm) — gate the outbound MCP +// tools/call. On a reflected reject, return SPARC's clarification as a +// JSON-RPC MCP tool *result*; the agent's MCP client consumes it like any +// tool output and asks the user for the missing detail. Robust to LLM +// streaming (reads the actual tool call, not the streamed response). +// - enforcement: "inference" — gate the agent's LLM response (where all three +// inputs are co-located). On a reflected reject, rewrite the completion so +// the assistant turn carries SPARC's clarification (tool_calls dropped). +// For agents that don't route tools through MCP. +// +// All enforcement policy lives here; the service only returns SPARC's verdict. +package sparc + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "path" + "strings" + "time" + + "github.com/kagenti/kagenti-extensions/authbridge/authlib/bypass" + "github.com/kagenti/kagenti-extensions/authbridge/authlib/pipeline" + "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins" +) + +// enforcement modes. +const ( + EnforcementMCP = "mcp" // gate the outbound MCP tools/call (default) + EnforcementInference = "inference" // gate the agent's LLM response (non-MCP agents) +) + +// on_reject_action values: what to do when SPARC returns "reject". +const ( + // OnRejectObserve records the reject and lets the call through (shadow mode). + OnRejectObserve = "observe" + // OnRejectReflect returns SPARC's reflection to the agent (MCP tool result + // in mcp mode, or a rewritten LLM response in inference mode) so the agent + // transparently re-asks the user. This is "reflection mode". + OnRejectReflect = "reflect" + // OnRejectDeny hard-blocks the call. + OnRejectDeny = "deny" +) + +// fail_policy values: what to do when SPARC is unavailable or returns "error". +const ( + FailOpen = "open" // allow + record (default — SPARC is a quality gate, not auth) + FailClosed = "closed" // block +) + +// sparcConfig is the plugin's local config schema. Field tags drive both +// runtime decoding (json) and operator-facing schema introspection +// (description / required / default / enum), matching ibac and the auth plugins. +// See authbridge/docs/sparc-plugin.md for prose. +type sparcConfig struct { + ReflectorEndpoint string `json:"reflector_endpoint" required:"true" description:"Base URL of the SPARC reflection service. The plugin POSTs to {endpoint}/reflect."` + + ReflectorBearer string `json:"reflector_bearer" description:"Optional bearer token for the reflection service. Empty for in-cluster unauthenticated calls."` + + Enforcement string `json:"enforcement" description:"Where to gate and how to return the verdict: mcp=gate the MCP tools/call, return an MCP tool result; inference=gate the LLM response, rewrite the completion." default:"mcp" enum:"mcp,inference"` + + Track string `json:"track" description:"SPARC reflection track." default:"fast_track" enum:"fast_track,slow_track,syntax,spec_free,transformations_only"` + + TimeoutMs int `json:"timeout_ms" description:"Per-call reflection timeout. SPARC LLM calls take seconds. Validation rejects values below 100." default:"30000"` + + OnRejectAction string `json:"on_reject_action" description:"Behavior on a SPARC reject: observe=log only; reflect=return SPARC's clarification to the agent; deny=hard block." default:"reflect" enum:"observe,reflect,deny"` + + DenyScoreThreshold float64 `json:"deny_score_threshold" description:"When SPARC's grounding score (0=worst..1=best) is <= this, escalate any reject to a hard deny regardless of on_reject_action. 0 disables escalation." default:"0"` + + FailPolicy string `json:"fail_policy" description:"On SPARC unavailable or decision=error: open=allow and record; closed=block." default:"open" enum:"open,closed"` + + SkipTools []string `json:"skip_tools" description:"Tool-name globs (path.Match) to NOT reflect on, e.g. trivial read-only tools. Evaluated before reflect_tools."` + + ReflectTools []string `json:"reflect_tools" description:"If non-empty, ONLY reflect on tools whose name matches one of these globs; all others are skipped."` + + BypassHosts []string `json:"bypass_hosts" description:"Host globs (path.Match) skipped without reflecting. Defaults include keycloak / spire / otel."` + + BypassPaths []string `json:"bypass_paths" description:"URL path globs skipped without reflecting. Defaults: /.well-known/* /healthz /readyz /livez."` +} + +// defaultBypassHosts mirrors ibac's conservative starting set (kept local so a +// future ibac tweak can't silently change sparc behavior). +var defaultBypassHosts = []string{ + "keycloak.*", "keycloak", "spire-server.*", "spire-agent.*", + "otel-collector.*", "jaeger.*", "prometheus.*", +} + +var defaultBypassPaths = []string{"/healthz", "/readyz", "/livez", "/.well-known/*"} + +func (c *sparcConfig) applyDefaults() { + if c.Enforcement == "" { + c.Enforcement = EnforcementMCP + } + if c.Track == "" { + c.Track = "fast_track" + } + if c.TimeoutMs == 0 { + c.TimeoutMs = 30000 + } + if c.OnRejectAction == "" { + c.OnRejectAction = OnRejectReflect + } + if c.FailPolicy == "" { + c.FailPolicy = FailOpen + } + if len(c.BypassHosts) == 0 { + c.BypassHosts = defaultBypassHosts + } + if len(c.BypassPaths) == 0 { + c.BypassPaths = defaultBypassPaths + } +} + +func (c *sparcConfig) validate() error { + if c.ReflectorEndpoint == "" { + return errors.New("reflector_endpoint is required") + } + if c.TimeoutMs < 100 { + return fmt.Errorf("timeout_ms must be at least 100, got %d", c.TimeoutMs) + } + switch c.Enforcement { + case EnforcementMCP, EnforcementInference: + default: + return fmt.Errorf("enforcement must be mcp or inference, got %q", c.Enforcement) + } + switch c.OnRejectAction { + case OnRejectObserve, OnRejectReflect, OnRejectDeny: + default: + return fmt.Errorf("on_reject_action must be observe|reflect|deny, got %q", c.OnRejectAction) + } + switch c.FailPolicy { + case FailOpen, FailClosed: + default: + return fmt.Errorf("fail_policy must be open or closed, got %q", c.FailPolicy) + } + if c.DenyScoreThreshold < 0 || c.DenyScoreThreshold > 1 { + return fmt.Errorf("deny_score_threshold must be in [0,1], got %v", c.DenyScoreThreshold) + } + for _, g := range []struct { + label string + pats []string + }{{"bypass_hosts", c.BypassHosts}, {"bypass_paths", c.BypassPaths}, {"skip_tools", c.SkipTools}, {"reflect_tools", c.ReflectTools}} { + for _, p := range g.pats { + if _, err := path.Match(p, ""); err != nil { + return fmt.Errorf("invalid %s pattern %q: %w", g.label, p, err) + } + } + } + for _, p := range c.BypassHosts { + if t := strings.TrimSpace(p); t == "" || t == "*" { + return fmt.Errorf("bypass_hosts pattern %q matches everything; remove sparc from the pipeline instead", p) + } + } + for _, p := range c.BypassPaths { + if t := strings.TrimSpace(p); t == "" || t == "*" || t == "/*" { + return fmt.Errorf("bypass_paths pattern %q matches everything; remove sparc from the pipeline instead", p) + } + } + return nil +} + +// SPARC reflects on proposed tool calls and enforces the configured policy. +type SPARC struct { + cfg sparcConfig + reflector Reflector + bypassPaths *bypass.Matcher + bypassHosts []string + timeout time.Duration +} + +// NewSPARC constructs an unconfigured plugin. Configure must be called first. +func NewSPARC() *SPARC { return &SPARC{} } + +func init() { + plugins.RegisterPlugin("sparc", func() pipeline.Plugin { return NewSPARC() }) +} + +func (p *SPARC) Name() string { return "sparc" } + +func (p *SPARC) Capabilities() pipeline.PluginCapabilities { + return pipeline.PluginCapabilities{ + // Needs a parser to supply SPARC's inputs. inference-parser provides the + // conversation + tool specs (both modes); mcp-parser provides the tool + // call (mcp mode). RequiresAny is a static "at least one" check; the + // per-mode runtime requirements are validated/handled below. + RequiresAny: []string{"inference-parser", "mcp-parser"}, + ReadsBody: true, + WritesBody: true, // MCP result (mcp mode) / completion rewrite (inference mode) + Description: "SPARC pre-tool reflection: blocks ungrounded/hallucinated tool calls.", + } +} + +func (p *SPARC) ConfigSchema() []pipeline.FieldSchema { + return pipeline.SchemaOf(sparcConfig{}) +} + +func (p *SPARC) Configure(raw json.RawMessage) error { + var c sparcConfig + if len(raw) > 0 { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&c); err != nil { + return fmt.Errorf("sparc config: %w", err) + } + } + c.applyDefaults() + if err := c.validate(); err != nil { + return fmt.Errorf("sparc config: %w", err) + } + p.cfg = c + + matcher, err := bypass.NewMatcher(c.BypassPaths) + if err != nil { + return fmt.Errorf("sparc bypass_paths: %w", err) + } + p.bypassPaths = matcher + p.bypassHosts = c.BypassHosts + p.timeout = time.Duration(c.TimeoutMs) * time.Millisecond + p.reflector = newHTTPReflector(c.ReflectorEndpoint, c.ReflectorBearer, p.timeout) + return nil +} + +// OnRequest handles mcp mode (gate the outbound MCP tools/call). +func (p *SPARC) OnRequest(ctx context.Context, pctx *pipeline.Context) pipeline.Action { + if p.reflector == nil { + return pipeline.DenyStatus(503, "sparc.unconfigured", "sparc plugin not configured") + } + if p.cfg.Enforcement != EnforcementMCP { + return cont() + } + if pctx.Headers.Get(sentinelHeader) == "1" { + return cont() + } + if p.bypassPaths.Match(pctx.Path) { + pctx.Skip("path_bypass") + return cont() + } + if matchesAnyHost(p.bypassHosts, pctx.Host) { + pctx.Skip("host_bypass") + return cont() + } + anyAction, anyBypass := pctx.Classification() + if anyBypass { + pctx.Skip("protocol_mechanics") + return cont() + } + if !anyAction { + return cont() + } + + toolName := extractMCPToolName(pctx.Extensions.MCP) + if toolName == "" { + pctx.Skip("not_tool_call") + return cont() + } + if p.toolSkipped(toolName) { + pctx.Skip("tool_skipped") + return cont() + } + + messages, toolSpecs, ok := correlateInferenceContext(pctx) + if !ok { + // No conversation/tool context to ground against. This is a missing + // prerequisite (no inference-parser event / session tracking off), not + // a verdict — apply fail_policy so a misconfigured pipeline doesn't + // silently run unprotected. + return p.handleNoContext(pctx, toolName, EnforcementMCP) + } + in := ReflectInput{ + Messages: messages, + ToolSpecs: toolSpecs, + ToolCalls: []map[string]any{buildToolCall(pctx.Extensions.MCP, toolName)}, + SessionID: sessionID(pctx), + Track: p.cfg.Track, + } + + verdict, err := p.reflector.Reflect(ctx, in) + if err != nil { + return p.handleUnavailable(pctx, toolName, err, EnforcementMCP) + } + p.emitEvent(pctx, toolName, verdict) + + switch verdict.Decision { + case DecisionApprove: + p.recordAllow(pctx, toolName, verdict, EnforcementMCP) + return cont() + case DecisionReject: + return p.handleReject(pctx, toolName, verdict, EnforcementMCP) + default: + return p.handleUnavailable(pctx, toolName, + fmt.Errorf("%w: sparc decision=%q", ErrReflectorUnavailable, verdict.Decision), EnforcementMCP) + } +} + +// OnResponse handles inference mode (gate the agent's LLM response). +func (p *SPARC) OnResponse(ctx context.Context, pctx *pipeline.Context) pipeline.Action { + if p.cfg.Enforcement != EnforcementInference { + return cont() + } + inf := pctx.Extensions.Inference + if inf == nil || len(inf.ToolCalls) == 0 { + return cont() // not an LLM tool-call response (e.g. streamed or plain text) + } + tc := inf.ToolCalls[0] + if tc.Name == "" { + return cont() + } + if p.toolSkipped(tc.Name) { + pctx.Skip("tool_skipped") + return cont() + } + + messages, toolSpecs, ok := inferenceInputs(inf) + if !ok { + return p.handleNoContext(pctx, tc.Name, EnforcementInference) + } + in := ReflectInput{ + Messages: messages, + ToolSpecs: toolSpecs, + ToolCalls: []map[string]any{openAIToolCall(tc.ID, tc.Name, tc.Arguments)}, + SessionID: sessionID(pctx), + Track: p.cfg.Track, + } + + verdict, err := p.reflector.Reflect(ctx, in) + if err != nil { + return p.handleUnavailable(pctx, tc.Name, err, EnforcementInference) + } + p.emitEvent(pctx, tc.Name, verdict) + + switch verdict.Decision { + case DecisionApprove: + p.recordAllow(pctx, tc.Name, verdict, EnforcementInference) + return cont() + case DecisionReject: + return p.handleReject(pctx, tc.Name, verdict, EnforcementInference) + default: + return p.handleUnavailable(pctx, tc.Name, + fmt.Errorf("%w: sparc decision=%q", ErrReflectorUnavailable, verdict.Decision), EnforcementInference) + } +} + +// handleReject applies on_reject_action with score-driven escalation. The +// returned action is format-aware per enforcement mode. +func (p *SPARC) handleReject(pctx *pipeline.Context, toolName string, verdict ReflectVerdict, mode string) pipeline.Action { + action := p.cfg.OnRejectAction + escalated := false + if p.cfg.DenyScoreThreshold > 0 && verdict.OverallAvgScore != nil && + *verdict.OverallAvgScore <= p.cfg.DenyScoreThreshold { + action = OnRejectDeny + escalated = true + } + details := map[string]string{ + "tool": toolName, "score": scoreString(verdict.OverallAvgScore), + "sparc": firstExplanation(verdict), "action": action, + "escalated": fmt.Sprintf("%t", escalated), + } + + switch action { + case OnRejectObserve: + details["mode"] = "observe" + pctx.Record(inv(pipeline.ActionObserve, mode, "reject_observed", details)) + return cont() + case OnRejectReflect: + pctx.Record(inv(pipeline.ActionModify, mode, "reflected", details)) + if mode == EnforcementInference { + rewriteInferenceResponse(pctx, buildClarificationText(verdict)) + return cont() + } + return mcpResultAction(mcpRPCID(pctx.Extensions.MCP), buildClarificationText(verdict)) + default: // OnRejectDeny + pctx.Record(inv(pipeline.ActionDeny, mode, "blocked", details)) + reason := firstExplanation(verdict) + if mode == EnforcementInference { + rewriteInferenceResponse(pctx, "Tool call blocked by policy: "+reason) + return cont() + } + return pipeline.DenyStatus(403, "sparc.blocked", reason) + } +} + +// handleUnavailable applies fail_policy when SPARC can't be reached or errored. +func (p *SPARC) handleUnavailable(pctx *pipeline.Context, toolName string, err error, mode string) pipeline.Action { + errPreview := preview(err.Error(), 240) + details := map[string]string{"tool": toolName, "error": errPreview} + if p.cfg.FailPolicy == FailOpen { + details["mode"] = "fail_open" + pctx.Record(inv(pipeline.ActionObserve, mode, "reflector_unavailable", details)) + slog.Warn("sparc: reflector unavailable, failing open", "host", pctx.Host, "error", errPreview) + return cont() + } + pctx.Record(inv(pipeline.ActionDeny, mode, "reflector_unavailable", details)) + slog.Warn("sparc: reflector unavailable, failing closed", "host", pctx.Host, "error", errPreview) + if mode == EnforcementInference { + rewriteInferenceResponse(pctx, "Tool call blocked: reflection service unavailable.") + return cont() + } + return pipeline.DenyStatus(503, "sparc.reflector_unavailable", errPreview) +} + +// handleNoContext applies fail_policy when the conversation/tool context needed +// to ground a tool call is missing (no inference-parser event, or session +// tracking off). fail_open skips and lets the call through — recorded so the gap +// is visible in the session timeline; fail_closed blocks rather than letting an +// unverifiable call past. Default-open behavior is unchanged from a plain skip. +func (p *SPARC) handleNoContext(pctx *pipeline.Context, toolName, mode string) pipeline.Action { + if p.cfg.FailPolicy == FailOpen { + pctx.Skip("no_inference_context") + return cont() + } + pctx.Record(inv(pipeline.ActionDeny, mode, "no_inference_context", map[string]string{"tool": toolName})) + slog.Warn("sparc: no inference context to ground against, failing closed", "host", pctx.Host, "tool", toolName) + if mode == EnforcementInference { + rewriteInferenceResponse(pctx, "Tool call blocked: no conversation context available to verify it.") + return cont() + } + return pipeline.DenyStatus(503, "sparc.no_inference_context", "no conversation context to verify the tool call") +} + +func (p *SPARC) recordAllow(pctx *pipeline.Context, toolName string, verdict ReflectVerdict, mode string) { + pctx.Record(inv(pipeline.ActionAllow, mode, "grounded", map[string]string{ + "tool": toolName, "score": scoreString(verdict.OverallAvgScore), + })) +} + +// emitEvent publishes the full SPARC verdict via the plugin-event escape-hatch +// so abctl / session consumers can render the structured reflection. +func (p *SPARC) emitEvent(pctx *pipeline.Context, toolName string, verdict ReflectVerdict) { + if pctx.Extensions.Custom == nil { + pctx.Extensions.Custom = map[string]any{} + } + pctx.Extensions.Custom["sparc"+pipeline.PluginEventSuffix] = sparcEvent{ + Tool: toolName, + Decision: verdict.Decision, + Score: verdict.OverallAvgScore, + Track: p.cfg.Track, + Enforcement: p.cfg.Enforcement, + Issues: verdict.Issues, + ExecutionMs: verdict.ExecutionTimeMs, + } +} + +// toolSkipped reports whether a tool is excluded from reflection by config. +func (p *SPARC) toolSkipped(name string) bool { + if anyGlobMatch(p.cfg.SkipTools, name) { + return true + } + if len(p.cfg.ReflectTools) > 0 && !anyGlobMatch(p.cfg.ReflectTools, name) { + return true + } + return false +} + +// sparcEvent is the structured reflection record surfaced to session consumers. +type sparcEvent struct { + Tool string `json:"tool"` + Decision string `json:"decision"` + Score *float64 `json:"score,omitempty"` + Track string `json:"track"` + Enforcement string `json:"enforcement"` + Issues []ReflectIssue `json:"issues,omitempty"` + ExecutionMs float64 `json:"executionMs,omitempty"` +} + +func cont() pipeline.Action { return pipeline.Action{Type: pipeline.Continue} } + +func inv(action pipeline.InvocationAction, mode, reason string, details map[string]string) pipeline.Invocation { + // mcp enforcement runs in OnRequest (request phase); inference enforcement + // runs in OnResponse (response phase). Tag the invocation accordingly so it + // lands in the right session-event snapshot. + phase := pipeline.InvocationPhaseRequest + if mode == EnforcementInference { + phase = pipeline.InvocationPhaseResponse + } + if mode != "" { + details["enforcement"] = mode + } + return pipeline.Invocation{ + Action: action, Phase: phase, Reason: reason, Details: details, + } +} + +// Compile-time interface checks. +var ( + _ pipeline.Plugin = (*SPARC)(nil) + _ pipeline.Configurable = (*SPARC)(nil) +) diff --git a/authbridge/authlib/plugins/sparc/plugin_test.go b/authbridge/authlib/plugins/sparc/plugin_test.go new file mode 100644 index 000000000..ee8414fbe --- /dev/null +++ b/authbridge/authlib/plugins/sparc/plugin_test.go @@ -0,0 +1,346 @@ +package sparc + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/kagenti/kagenti-extensions/authbridge/authlib/pipeline" +) + +type fakeReflector struct { + verdict ReflectVerdict + err error + gotIn ReflectInput + calls int +} + +func (f *fakeReflector) Reflect(_ context.Context, in ReflectInput) (ReflectVerdict, error) { + f.calls++ + f.gotIn = in + return f.verdict, f.err +} + +func configured(t *testing.T, cfgJSON string, fr *fakeReflector) *SPARC { + t.Helper() + p := NewSPARC() + if cfgJSON == "" { + cfgJSON = `{"reflector_endpoint":"http://sparc.invalid","timeout_ms":1000}` + } + if err := p.Configure(json.RawMessage(cfgJSON)); err != nil { + t.Fatalf("Configure: %v", err) + } + if fr != nil { + p.reflector = fr + } + return p +} + +func invokeReq(p pipeline.Plugin, pctx *pipeline.Context) pipeline.Action { + pctx.SetCurrentPlugin(p.Name(), pipeline.InvocationPhaseRequest) + defer pctx.ClearCurrentPlugin() + return p.OnRequest(context.Background(), pctx) +} + +func invokeResp(p pipeline.Plugin, pctx *pipeline.Context) pipeline.Action { + pctx.SetCurrentPlugin(p.Name(), pipeline.InvocationPhaseResponse) + defer pctx.ClearCurrentPlugin() + return p.OnResponse(context.Background(), pctx) +} + +// inferenceEvent builds a session inference request event (the generic source +// of conversation + tool specs). +func inferenceEvent() pipeline.SessionEvent { + return pipeline.SessionEvent{ + Direction: pipeline.Outbound, Phase: pipeline.SessionRequest, + Inference: &pipeline.InferenceExtension{ + Model: "llama3.2:3b", + Messages: []pipeline.InferenceMessage{ + {Role: "system", Content: "You are a finance assistant."}, + {Role: "user", Content: "Refund my duplicate charge."}, + }, + Tools: []pipeline.InferenceTool{ + {Name: "issue_refund", Description: "Issue a refund", Parameters: map[string]any{"type": "object"}}, + }, + }, + } +} + +// mcpPctx builds an outbound pctx for mcp mode: an action-classified MCP +// tools/call plus a session carrying one inference request event. +func mcpPctx() *pipeline.Context { + pctx := &pipeline.Context{ + Direction: pipeline.Outbound, Method: "POST", Scheme: "http", + Host: "finance-mcp.team1.svc", Path: "/mcp", Headers: http.Header{}, + Session: &pipeline.SessionView{ID: "s1", Events: []pipeline.SessionEvent{inferenceEvent()}}, + } + pctx.Extensions.MCP = &pipeline.MCPExtension{ + Method: "tools/call", RPCID: "rpc-1", IsAction: true, + Params: map[string]any{"name": "issue_refund", "arguments": map[string]any{"transaction_id": "TX9999"}}, + } + return pctx +} + +// inferencePctx builds a pctx for inference mode: the LLM response carries a +// proposed tool call plus the request messages+tools. +func inferencePctx() *pipeline.Context { + pctx := &pipeline.Context{ + Direction: pipeline.Outbound, Method: "POST", Scheme: "http", + Host: "ollama", Path: "/v1/chat/completions", Headers: http.Header{}, + ResponseBody: []byte(`{"id":"x","object":"chat.completion","model":"m","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"c1","type":"function","function":{"name":"issue_refund","arguments":"{\"transaction_id\":\"TX9999\"}"}}]},"finish_reason":"tool_calls"}]}`), + } + inf := inferenceEvent().Inference + inf.ToolCalls = []pipeline.InferenceToolCall{{ID: "c1", Name: "issue_refund", Arguments: `{"transaction_id":"TX9999"}`}} + pctx.Extensions.Inference = inf + return pctx +} + +func outInvs(pctx *pipeline.Context) []pipeline.Invocation { + if pctx.Extensions.Invocations == nil { + return nil + } + return pctx.Extensions.Invocations.Outbound +} + +// --- Configure --- + +func TestConfigure_RequiresEndpoint(t *testing.T) { + if err := NewSPARC().Configure(json.RawMessage(`{}`)); err == nil { + t.Fatal("expected error when reflector_endpoint missing") + } +} + +func TestConfigure_Defaults(t *testing.T) { + p := configured(t, `{"reflector_endpoint":"http://x"}`, nil) + if p.cfg.Enforcement != EnforcementMCP || p.cfg.Track != "fast_track" || + p.cfg.OnRejectAction != OnRejectReflect || p.cfg.FailPolicy != FailOpen || p.cfg.TimeoutMs != 30000 { + t.Fatalf("defaults not applied: %+v", p.cfg) + } +} + +func TestConfigure_RejectsBadEnums(t *testing.T) { + for _, bad := range []string{ + `{"reflector_endpoint":"http://x","enforcement":"nope"}`, + `{"reflector_endpoint":"http://x","on_reject_action":"nope"}`, + `{"reflector_endpoint":"http://x","fail_policy":"maybe"}`, + `{"reflector_endpoint":"http://x","deny_score_threshold":9}`, + `{"reflector_endpoint":"http://x","bogus":1}`, + } { + if err := NewSPARC().Configure(json.RawMessage(bad)); err == nil { + t.Errorf("expected error for %s", bad) + } + } +} + +// --- mcp mode --- + +func TestMCP_ApproveContinues(t *testing.T) { + score := 4.5 + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionApprove, OverallAvgScore: &score}} + p := configured(t, "", fr) + pctx := mcpPctx() + if act := invokeReq(p, pctx); act.Type != pipeline.Continue { + t.Fatalf("approve should Continue, got %+v", act) + } + // generic collection: system prompt + tools + tool_call all forwarded + if len(fr.gotIn.Messages) != 2 || fr.gotIn.Messages[0]["role"] != "system" { + t.Errorf("expected system+user messages, got %+v", fr.gotIn.Messages) + } + if len(fr.gotIn.ToolSpecs) != 1 || len(fr.gotIn.ToolCalls) != 1 { + t.Errorf("expected tool_specs + tool_call, got %+v", fr.gotIn) + } + if fn := fr.gotIn.ToolCalls[0]["function"].(map[string]any); fn["name"] != "issue_refund" { + t.Errorf("tool_call name = %v", fn["name"]) + } + if i := outInvs(pctx); len(i) != 1 || i[0].Action != pipeline.ActionAllow { + t.Errorf("expected one allow, got %+v", i) + } +} + +func TestMCP_RejectReflects(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject, Issues: []ReflectIssue{{Explanation: "TX9999 not grounded"}}}} + p := configured(t, `{"reflector_endpoint":"http://x","on_reject_action":"reflect"}`, fr) + pctx := mcpPctx() + act := invokeReq(p, pctx) + if act.Type != pipeline.Reject || act.Violation == nil || act.Violation.Status != 200 { + t.Fatalf("reflect should be Reject w/ 200 MCP result, got %+v", act) + } + var env map[string]any + if err := json.Unmarshal(act.Violation.Body, &env); err != nil || env["result"] == nil { + t.Fatalf("body is not an MCP result: %v / %v", err, env) + } + if i := outInvs(pctx); len(i) != 1 || i[0].Action != pipeline.ActionModify { + t.Errorf("expected modify, got %+v", i) + } + // structured event emitted + if _, ok := pctx.Extensions.Custom["sparc"+pipeline.PluginEventSuffix]; !ok { + t.Errorf("expected sparc event in Custom") + } +} + +func TestMCP_RejectObserveContinues(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject, Issues: []ReflectIssue{{Explanation: "x"}}}} + p := configured(t, `{"reflector_endpoint":"http://x","on_reject_action":"observe"}`, fr) + pctx := mcpPctx() + if act := invokeReq(p, pctx); act.Type != pipeline.Continue { + t.Fatalf("observe should Continue, got %+v", act) + } + if i := outInvs(pctx); len(i) != 1 || i[0].Action != pipeline.ActionObserve { + t.Errorf("expected observe, got %+v", i) + } +} + +func TestMCP_RejectDeny(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject, Issues: []ReflectIssue{{Explanation: "blocked"}}}} + p := configured(t, `{"reflector_endpoint":"http://x","on_reject_action":"deny"}`, fr) + act := invokeReq(p, mcpPctx()) + if act.Type != pipeline.Reject || act.Violation == nil || act.Violation.Code != "sparc.blocked" { + t.Fatalf("deny should Reject sparc.blocked, got %+v", act) + } +} + +func TestMCP_ScoreEscalatesToDeny(t *testing.T) { + low := 0.5 + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject, OverallAvgScore: &low, Issues: []ReflectIssue{{Explanation: "severe"}}}} + p := configured(t, `{"reflector_endpoint":"http://x","on_reject_action":"reflect","deny_score_threshold":0.7}`, fr) + act := invokeReq(p, mcpPctx()) + if act.Type != pipeline.Reject || act.Violation == nil || act.Violation.Code != "sparc.blocked" { + t.Fatalf("low score should escalate to deny, got %+v", act) + } +} + +func TestMCP_SkipTools(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject}} + p := configured(t, `{"reflector_endpoint":"http://x","skip_tools":["issue_*"]}`, fr) + if act := invokeReq(p, mcpPctx()); act.Type != pipeline.Continue { + t.Fatalf("skipped tool should Continue, got %+v", act) + } + if fr.calls != 0 { + t.Errorf("reflector should not be called for skipped tool") + } +} + +func TestMCP_ReflectToolsAllowlist(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionApprove}} + p := configured(t, `{"reflector_endpoint":"http://x","reflect_tools":["get_*"]}`, fr) + // issue_refund not in allow-list → skipped + if act := invokeReq(p, mcpPctx()); act.Type != pipeline.Continue || fr.calls != 0 { + t.Fatalf("non-allowlisted tool should be skipped, got act=%+v calls=%d", act, fr.calls) + } +} + +func TestMCP_NoInferenceContextFailsOpen(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject}} + p := configured(t, "", fr) // default fail_policy: open + pctx := mcpPctx() + pctx.Session = &pipeline.SessionView{ID: "s1"} // no inference event + if act := invokeReq(p, pctx); act.Type != pipeline.Continue { + t.Fatalf("no context (fail_open) should Continue, got %+v", act) + } + if fr.calls != 0 { + t.Errorf("reflector should not be called without context") + } +} + +func TestMCP_NoInferenceContextFailsClosed(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject}} + p := configured(t, `{"reflector_endpoint":"http://x","fail_policy":"closed"}`, fr) + pctx := mcpPctx() + pctx.Session = &pipeline.SessionView{ID: "s1"} // no inference event + act := invokeReq(p, pctx) + if act.Type != pipeline.Reject || act.Violation == nil || act.Violation.Status != 503 { + t.Fatalf("no context (fail_closed) should 503, got %+v", act) + } + if fr.calls != 0 { + t.Errorf("reflector should not be called without context") + } +} + +func TestMCP_NotAToolCallSkips(t *testing.T) { + fr := &fakeReflector{} + p := configured(t, "", fr) + pctx := mcpPctx() + pctx.Extensions.MCP.Method = "prompts/get" + if act := invokeReq(p, pctx); act.Type != pipeline.Continue || fr.calls != 0 { + t.Fatalf("non tools/call should skip, got act=%+v calls=%d", act, fr.calls) + } +} + +func TestMCP_Reentrancy(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject}} + p := configured(t, "", fr) + pctx := mcpPctx() + pctx.Headers.Set(sentinelHeader, "1") + if act := invokeReq(p, pctx); act.Type != pipeline.Continue || fr.calls != 0 { + t.Fatalf("reentrant call should Continue without reflecting") + } +} + +func TestMCP_FailOpenAndClosed(t *testing.T) { + frOpen := &fakeReflector{err: ErrReflectorUnavailable} + if act := invokeReq(configured(t, `{"reflector_endpoint":"http://x","fail_policy":"open"}`, frOpen), mcpPctx()); act.Type != pipeline.Continue { + t.Fatalf("fail_open should Continue, got %+v", act) + } + frClosed := &fakeReflector{err: ErrReflectorUnavailable} + act := invokeReq(configured(t, `{"reflector_endpoint":"http://x","fail_policy":"closed"}`, frClosed), mcpPctx()) + if act.Type != pipeline.Reject || act.Violation.Status != 503 { + t.Fatalf("fail_closed should 503, got %+v", act) + } +} + +// --- inference mode --- + +func TestInference_RejectRewritesResponse(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject, Issues: []ReflectIssue{{Explanation: "ungrounded id"}}}} + p := configured(t, `{"reflector_endpoint":"http://x","enforcement":"inference","on_reject_action":"reflect"}`, fr) + pctx := inferencePctx() + if act := invokeResp(p, pctx); act.Type != pipeline.Continue { + t.Fatalf("inference reject should Continue (body rewritten), got %+v", act) + } + // response body rewritten: tool_calls dropped, clarification in content + var out map[string]any + if err := json.Unmarshal(pctx.ResponseBody, &out); err != nil { + t.Fatalf("rewritten body invalid: %v", err) + } + msg := out["choices"].([]any)[0].(map[string]any)["message"].(map[string]any) + if _, hasTC := msg["tool_calls"]; hasTC { + t.Errorf("tool_calls should be dropped, got %v", msg) + } + if c, _ := msg["content"].(string); c == "" { + t.Errorf("clarification content missing") + } + // generic collection: all 3 inputs came straight from the inference ext + if len(fr.gotIn.Messages) != 2 || len(fr.gotIn.ToolSpecs) != 1 || len(fr.gotIn.ToolCalls) != 1 { + t.Errorf("expected 3 inputs co-located, got %+v", fr.gotIn) + } +} + +func TestInference_ApproveLeavesResponse(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionApprove}} + p := configured(t, `{"reflector_endpoint":"http://x","enforcement":"inference"}`, fr) + pctx := inferencePctx() + orig := string(pctx.ResponseBody) + invokeResp(p, pctx) + if string(pctx.ResponseBody) != orig { + t.Errorf("approve should not modify the response") + } +} + +func TestInference_MCPModeOnResponseIsNoop(t *testing.T) { + fr := &fakeReflector{verdict: ReflectVerdict{Decision: DecisionReject}} + p := configured(t, "", fr) // mcp mode + if act := invokeResp(p, inferencePctx()); act.Type != pipeline.Continue || fr.calls != 0 { + t.Fatalf("mcp-mode OnResponse should be a no-op") + } +} + +func TestCapabilities(t *testing.T) { + caps := NewSPARC().Capabilities() + if !caps.WritesBody || !caps.ReadsBody { + t.Error("expected ReadsBody+WritesBody") + } + if len(caps.RequiresAny) == 0 { + t.Error("expected RequiresAny parsers") + } +} diff --git a/authbridge/authlib/plugins/sparc/reflector.go b/authbridge/authlib/plugins/sparc/reflector.go new file mode 100644 index 000000000..221923e7f --- /dev/null +++ b/authbridge/authlib/plugins/sparc/reflector.go @@ -0,0 +1,129 @@ +package sparc + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" +) + +// Reflector evaluates whether a proposed tool call is grounded in the +// conversation and tool specifications. The concrete implementation +// (httpReflector) calls the out-of-process SPARC reflection service; tests +// inject a fake. +// +// A returned error means the reflection could not be obtained (transport +// failure, timeout, non-2xx). It is distinct from a successful reflection +// whose Decision is "error" (SPARC ran but its own LLM/pipeline failed) — +// the plugin maps both to the configured fail policy, but keeps them +// distinguishable in session records. +type Reflector interface { + Reflect(ctx context.Context, in ReflectInput) (ReflectVerdict, error) +} + +// ReflectInput is SPARC's native input: the conversation, the available +// tools, and the proposed tool call(s) to vet. +type ReflectInput struct { + Messages []map[string]any `json:"messages"` + ToolSpecs []map[string]any `json:"tool_specs"` + ToolCalls []map[string]any `json:"tool_calls"` + SessionID string `json:"session_id,omitempty"` + Track string `json:"track,omitempty"` +} + +// ReflectIssue mirrors one SPARC issue (PyPI agent-lifecycle-toolkit 0.10.1 +// shape: issue_type / metric_name / explanation / correction). +type ReflectIssue struct { + IssueType string `json:"issue_type"` + MetricName string `json:"metric_name"` + Explanation string `json:"explanation"` + Correction any `json:"correction"` +} + +// ReflectVerdict is SPARC's decision plus its supporting issues and score. +type ReflectVerdict struct { + Decision string `json:"decision"` // approve | reject | error + Issues []ReflectIssue `json:"issues"` + OverallAvgScore *float64 `json:"overall_avg_score"` // 0=worst .. 1=best; nil when unavailable + ExecutionTimeMs float64 `json:"execution_time_ms"` +} + +// SPARC decision values. +const ( + DecisionApprove = "approve" + DecisionReject = "reject" + DecisionError = "error" +) + +// ErrReflectorUnavailable is returned (wrapped) when the SPARC service could +// not be reached or returned a non-2xx status — an availability problem, +// distinct from a "reject" verdict. +var ErrReflectorUnavailable = errors.New("sparc reflector unavailable") + +// sentinelHeader marks the plugin's own outbound HTTP call so that, if it +// ever loops back through the listener, OnRequest short-circuits instead of +// recursing (the same defense-in-depth pattern ibac's judge uses). +const sentinelHeader = "X-SPARC-Reflect" + +// httpReflector POSTs a reflection request to {endpoint}/reflect. +// +// The call is made with a standalone http.Client that does NOT route back +// through the authbridge listener, structurally preventing reentrancy. The +// sentinel header is belt-and-suspenders. +type httpReflector struct { + url string + bearer string + client *http.Client + timeout time.Duration +} + +func newHTTPReflector(endpoint, bearer string, timeout time.Duration) *httpReflector { + return &httpReflector{ + url: endpoint + "/reflect", + bearer: bearer, + client: &http.Client{Timeout: timeout}, + timeout: timeout, + } +} + +func (r *httpReflector) Reflect(ctx context.Context, in ReflectInput) (ReflectVerdict, error) { + body, err := json.Marshal(in) + if err != nil { + return ReflectVerdict{}, fmt.Errorf("marshal reflect request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.url, bytes.NewReader(body)) + if err != nil { + return ReflectVerdict{}, fmt.Errorf("%w: %v", ErrReflectorUnavailable, err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set(sentinelHeader, "1") + if r.bearer != "" { + req.Header.Set("Authorization", "Bearer "+r.bearer) + } + + resp, err := r.client.Do(req) + if err != nil { + return ReflectVerdict{}, fmt.Errorf("%w: %v", ErrReflectorUnavailable, err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return ReflectVerdict{}, fmt.Errorf("%w: status %d: %s", + ErrReflectorUnavailable, resp.StatusCode, preview(string(respBody), 240)) + } + + var v ReflectVerdict + if err := json.Unmarshal(respBody, &v); err != nil { + return ReflectVerdict{}, fmt.Errorf("%w: decode response: %v", ErrReflectorUnavailable, err) + } + if v.Decision == "" { + return ReflectVerdict{}, fmt.Errorf("%w: empty decision in response", ErrReflectorUnavailable) + } + return v, nil +} diff --git a/authbridge/authlib/plugins/sparc/reflector_test.go b/authbridge/authlib/plugins/sparc/reflector_test.go new file mode 100644 index 000000000..55a6c09db --- /dev/null +++ b/authbridge/authlib/plugins/sparc/reflector_test.go @@ -0,0 +1,92 @@ +package sparc + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestHTTPReflector_DecodesVerdict(t *testing.T) { + var gotPath, gotSentinel string + var gotBody ReflectInput + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotSentinel = r.Header.Get(sentinelHeader) + _ = json.NewDecoder(r.Body).Decode(&gotBody) + score := 2.0 + _ = json.NewEncoder(w).Encode(ReflectVerdict{ + Decision: DecisionReject, + OverallAvgScore: &score, + Issues: []ReflectIssue{ + {IssueType: "semantic_function", MetricName: "m", Explanation: "ungrounded id"}, + }, + }) + })) + defer srv.Close() + + r := newHTTPReflector(srv.URL, "", time.Second) + v, err := r.Reflect(context.Background(), ReflectInput{ + ToolCalls: []map[string]any{{"function": map[string]any{"name": "get_transaction"}}}, + SessionID: "s1", + }) + if err != nil { + t.Fatalf("Reflect: %v", err) + } + if gotPath != "/reflect" { + t.Errorf("path = %q, want /reflect", gotPath) + } + if gotSentinel != "1" { + t.Errorf("sentinel header = %q, want 1", gotSentinel) + } + if gotBody.SessionID != "s1" { + t.Errorf("forwarded session_id = %q, want s1", gotBody.SessionID) + } + if v.Decision != DecisionReject { + t.Errorf("decision = %q, want reject", v.Decision) + } + if v.OverallAvgScore == nil || *v.OverallAvgScore != 2.0 { + t.Errorf("score = %v, want 2.0", v.OverallAvgScore) + } + if len(v.Issues) != 1 || v.Issues[0].Explanation != "ungrounded id" { + t.Errorf("issues = %+v", v.Issues) + } +} + +func TestHTTPReflector_Non2xxIsUnavailable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, `{"error":"boom"}`, http.StatusBadGateway) + })) + defer srv.Close() + + r := newHTTPReflector(srv.URL, "", time.Second) + _, err := r.Reflect(context.Background(), ReflectInput{ToolCalls: []map[string]any{{}}}) + if !errors.Is(err, ErrReflectorUnavailable) { + t.Fatalf("expected ErrReflectorUnavailable, got %v", err) + } +} + +func TestHTTPReflector_EmptyDecisionIsUnavailable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"issues":[]}`)) + })) + defer srv.Close() + + r := newHTTPReflector(srv.URL, "", time.Second) + _, err := r.Reflect(context.Background(), ReflectInput{ToolCalls: []map[string]any{{}}}) + if !errors.Is(err, ErrReflectorUnavailable) { + t.Fatalf("expected ErrReflectorUnavailable for empty decision, got %v", err) + } +} + +func TestHTTPReflector_TransportErrorIsUnavailable(t *testing.T) { + // Point at a closed port to force a dial failure. + r := newHTTPReflector("http://127.0.0.1:1", "", 200*time.Millisecond) + _, err := r.Reflect(context.Background(), ReflectInput{ToolCalls: []map[string]any{{}}}) + if !errors.Is(err, ErrReflectorUnavailable) { + t.Fatalf("expected ErrReflectorUnavailable, got %v", err) + } +} diff --git a/authbridge/authlib/plugins/sparc/respond.go b/authbridge/authlib/plugins/sparc/respond.go new file mode 100644 index 000000000..7cfaaca5d --- /dev/null +++ b/authbridge/authlib/plugins/sparc/respond.go @@ -0,0 +1,123 @@ +package sparc + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/kagenti/kagenti-extensions/authbridge/authlib/pipeline" +) + +// buildClarificationText renders an agent-facing message from SPARC's issues. +// The agent receives this (as a tool result or rewritten completion) and relays +// it — typically asking the user for the missing / corrected information — +// without learning that SPARC intervened. +func buildClarificationText(verdict ReflectVerdict) string { + var b strings.Builder + b.WriteString("This tool call was not executed because it could not be verified against the conversation. ") + if len(verdict.Issues) == 0 { + b.WriteString("Please confirm the request details with the user before retrying.") + return b.String() + } + b.WriteString("Reason(s):") + for _, iss := range verdict.Issues { + if iss.Explanation == "" { + continue + } + fmt.Fprintf(&b, "\n- %s", iss.Explanation) + if c := correctionHint(iss.Correction); c != "" { + fmt.Fprintf(&b, " (suggested fix: %s)", c) + } + } + b.WriteString("\nAsk the user for the exact, missing, or corrected information, then retry.") + return b.String() +} + +func correctionHint(correction any) string { + if correction == nil { + return "" + } + if s, ok := correction.(string); ok { + return s + } + b, err := json.Marshal(correction) + if err != nil { + return "" + } + return preview(string(b), 200) +} + +// --- mcp mode: synthetic JSON-RPC MCP tool result --- + +// buildMCPResultBody crafts a JSON-RPC 2.0 MCP tools/call result envelope that +// carries the clarification as tool output. Returned with HTTP 200, the agent's +// MCP client consumes it as a normal tool result. +func buildMCPResultBody(rpcID any, text string) ([]byte, error) { + if rpcID == nil { + rpcID = 0 + } + return json.Marshal(map[string]any{ + "jsonrpc": "2.0", + "id": rpcID, + "result": map[string]any{ + "content": []map[string]any{{"type": "text", "text": text}}, + "isError": false, + "_meta": map[string]any{"sparc": map[string]any{"reflected": true}}, + }, + }) +} + +// mcpResultAction returns a Reject whose Violation renders as an HTTP 200 MCP +// result body. Reject is the only pipeline primitive that short-circuits the +// outbound call and synthesizes a response; the 200 + JSON-RPC result body turn +// that into a normal tool result. +func mcpResultAction(rpcID any, text string) pipeline.Action { + body, err := buildMCPResultBody(rpcID, text) + if err != nil { + return pipeline.DenyStatus(403, "sparc.blocked", text) + } + return pipeline.Action{ + Type: pipeline.Reject, + Violation: &pipeline.Violation{ + Code: "sparc.reflected", + Reason: "tool call replaced with a SPARC clarification result", + Status: 200, + Body: body, + BodyType: "application/json", + }, + } +} + +// --- inference mode: rewrite the OpenAI chat completion --- + +// rewriteInferenceResponse replaces the model's tool-call turn with a plain +// assistant message carrying `text`, in OpenAI chat-completion shape, via +// SetResponseBody. The agent receives a normal completion (no tool_call) and +// relays the clarification to the user. Preserves id/model/object from the +// upstream response when present. +func rewriteInferenceResponse(pctx *pipeline.Context, text string) { + assistant := map[string]any{"role": "assistant", "content": text} + + var orig map[string]any + if err := json.Unmarshal(pctx.ResponseBody, &orig); err == nil { + if choices, ok := orig["choices"].([]any); ok && len(choices) > 0 { + if c0, ok := choices[0].(map[string]any); ok { + c0["message"] = assistant + c0["finish_reason"] = "stop" + delete(c0, "delta") + choices[0] = c0 + orig["choices"] = choices[:1] + if out, err := json.Marshal(orig); err == nil { + pctx.SetResponseBody(out) + return + } + } + } + } + // Fallback: synthesize a minimal OpenAI completion. + out, _ := json.Marshal(map[string]any{ + "object": "chat.completion", + "choices": []map[string]any{{"index": 0, "message": assistant, "finish_reason": "stop"}}, + }) + pctx.SetResponseBody(out) +} diff --git a/authbridge/cmd/authbridge-envoy/main.go b/authbridge/cmd/authbridge-envoy/main.go index 7b995675c..8ebed98ee 100644 --- a/authbridge/cmd/authbridge-envoy/main.go +++ b/authbridge/cmd/authbridge-envoy/main.go @@ -53,6 +53,7 @@ import ( _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/inferenceparser" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/jwtvalidation" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/mcpparser" + _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/sparc" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/tokenbroker" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/tokenexchange" ) diff --git a/authbridge/cmd/authbridge-proxy/main.go b/authbridge/cmd/authbridge-proxy/main.go index 21d7b6865..5a4614b47 100644 --- a/authbridge/cmd/authbridge-proxy/main.go +++ b/authbridge/cmd/authbridge-proxy/main.go @@ -46,6 +46,7 @@ import ( _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/inferenceparser" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/jwtvalidation" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/mcpparser" + _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/sparc" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/tokenbroker" _ "github.com/kagenti/kagenti-extensions/authbridge/authlib/plugins/tokenexchange" ) diff --git a/authbridge/demos/README.md b/authbridge/demos/README.md index 9a8900332..389873436 100644 --- a/authbridge/demos/README.md +++ b/authbridge/demos/README.md @@ -22,6 +22,7 @@ more AuthBridge capabilities. | **[MCP Parser Plugin](mcp-parser/README.md)** | Reference | Enable the `mcp-parser` plugin to surface tool calls / resource reads in session events | Configuration only | | **[abctl Walkthrough](weather-agent/demo-with-abctl.md)** | Reference | Watch the AuthBridge plugin pipeline live with the `abctl` TUI | Tooling only | | **[IBAC](ibac/README.md)** | Intermediate | Intent-Based Access Control: LLM judge denies outbound HTTP that doesn't align with the user's recorded intent. Reproduces the email-poison / prompt-injection attack from `huang195/ibac`; chat with the agent through the kagenti UI and see the exfiltration blocked, then `make show-result` for a pipeline-level forensic | UI + kubectl | +| **[SPARC (finance)](finance-sparc/README.md)** | Intermediate | SPARC pre-tool reflection: the `sparc` plugin blocks a hallucinated/ungrounded tool argument (an invented transaction id) before it executes and transparently asks the user to clarify, then approves the corrected call. Complements IBAC — SPARC verifies argument grounding, IBAC verifies intent alignment | UI + kubectl | ## Recommended Path diff --git a/authbridge/demos/finance-sparc/.gitignore b/authbridge/demos/finance-sparc/.gitignore new file mode 100644 index 000000000..ab6901c56 --- /dev/null +++ b/authbridge/demos/finance-sparc/.gitignore @@ -0,0 +1 @@ +.stamps/ diff --git a/authbridge/demos/finance-sparc/Makefile b/authbridge/demos/finance-sparc/Makefile new file mode 100644 index 000000000..fe645499f --- /dev/null +++ b/authbridge/demos/finance-sparc/Makefile @@ -0,0 +1,213 @@ +# finance-sparc demo — SPARC pre-tool reflection on a regular finance agent. +# +# make demo # full end-to-end, watsonx (needs WX_* env) +# make demo PROVIDER=ollama # full end-to-end, local ollama (no creds) +# make undeploy # remove demo resources +# +# `make demo` is one-shot: it sets up the host (ollama reachable from the +# cluster), builds + loads images (incl. an authbridge image carrying the sparc +# plugin), deploys the SPARC service + finance MCP server + agent, enables the +# pipeline, configures Keycloak, and drives the two-turn scenario through REAL +# inbound auth (no jwt bypass), printing SPARC's verdicts. +# +# Prereqs: a kagenti kind cluster, Ollama on the host with the model, and +# kubectl / kind / docker(or podman) / python3 / uv. For PROVIDER=watsonx also +# export WX_API_KEY and WX_PROJECT_ID (e.g. `set -a; . /path/.env; set +a`). + +.PHONY: help preflight host-setup build-images build-authbridge load-images \ + ollama-up sparc-config deploy wait-pods patch-config setup-keycloak drive demo \ + build-abctl show-result undeploy logs-agent logs-sparc logs-ollama + +.DEFAULT_GOAL := help + +PROVIDER ?= watsonx +KIND_CLUSTER_NAME ?= kagenti +KIND_NODE := $(KIND_CLUSTER_NAME)-control-plane +# NS / AGENT / PLATFORM_NS are fixed: the k8s/ manifests hardcode these names +# (team1 / finance-agent / kagenti-system), so they are not override knobs — +# changing them here alone would target resources the manifests never create. +NS := team1 +AGENT := finance-agent +PLATFORM_NS := kagenti-system +# Prefer whichever runtime's daemon is actually reachable (docker first). +CONTAINER_RUNTIME ?= $(shell docker info >/dev/null 2>&1 && echo docker || (podman info >/dev/null 2>&1 && echo podman) || echo docker) +AUTHBRIDGE_IMAGE ?= ghcr.io/kagenti/kagenti-extensions/authbridge:v0.6.0-alpha.4 + +AGENT_IMAGE := finance-agent:latest +MCP_IMAGE := finance-mcp:latest +SPARC_IMAGE := sparc-service:latest +# Pinned (must match k8s/ollama.yaml). Override only if you stage a different tag. +OLLAMA_IMAGE := ollama/ollama:0.30.2 +# In-cluster Ollama (the agent's reasoning LLM for the watsonx path). +OLLAMA_URL := http://ollama.$(NS).svc.cluster.local:11434 + +# Host Ollama, reachable from the cluster. ALTK's ollama reflection is free-form +# generate+validate (guided decoding is disabled for ollama), which is slow on a +# CPU-only in-cluster node but fast on a Metal/GPU host. So PROVIDER=ollama uses +# the HOST Ollama for both reasoning and reflection. host.docker.internal is the +# portable default; on Rancher Desktop/colima use http://192.168.5.2:11434. +HOST_OLLAMA_URL ?= http://host.docker.internal:11434 + +# Per-provider models/timeouts (all override-able). +# - watsonx: reflection on watsonx (fast); reasoning on in-cluster Ollama (1b). +# - ollama: reflection AND reasoning on the host Ollama. Reflection needs a +# capable instruct model (3b/1b hallucinate the function-selection +# metric); reasoning needs a clean tool-caller. Override SPARC_MODEL/ +# AGENT_OLLAMA_MODEL to match what your host Ollama has pulled. +ifeq ($(PROVIDER),watsonx) +SPARC_MODEL ?= mistral-large-2512 +SPARC_TIMEOUT_MS ?= 30000 +TURN_TIMEOUT ?= 240 +AGENT_OLLAMA_URL ?= $(OLLAMA_URL) +AGENT_OLLAMA_MODEL ?= llama3.2:1b +else +SPARC_OLLAMA_URL ?= $(HOST_OLLAMA_URL) +SPARC_MODEL ?= gemma4:e4b +SPARC_TIMEOUT_MS ?= 200000 +TURN_TIMEOUT ?= 600 +AGENT_OLLAMA_URL ?= $(HOST_OLLAMA_URL) +AGENT_OLLAMA_MODEL ?= llama3.2:3b +endif + +help: ## Show this menu + @printf "\nfinance-sparc demo — SPARC reflection on a regular finance agent.\n\n" + @printf "Run:\n \033[1mmake demo\033[0m (watsonx; needs WX_* env)\n" + @printf " \033[1mmake demo PROVIDER=ollama\033[0m (local ollama; no creds)\n\n" + @printf "Targets:\n" + @awk 'BEGIN{FS=":.*## "} /^[a-zA-Z_-]+:.*## /{printf " \033[36m%-16s\033[0m %s\n",$$1,$$2}' $(MAKEFILE_LIST) + @printf "\nVars: PROVIDER(=%s) KIND_CLUSTER_NAME AUTHBRIDGE_IMAGE SPARC_MODEL HOST_OLLAMA_URL\n\n" "$(PROVIDER)" + +preflight: ## Verify kagenti + tools (+ watsonx creds when PROVIDER=watsonx) + @kubectl get ns $(PLATFORM_NS) >/dev/null 2>&1 || { echo "ERROR: '$(PLATFORM_NS)' not found — install kagenti first."; exit 1; } + @kubectl get ns $(NS) >/dev/null 2>&1 || { echo "ERROR: agent namespace '$(NS)' not found."; exit 1; } + @for t in kubectl kind python3 uv; do command -v $$t >/dev/null 2>&1 || { echo "ERROR: $$t not installed"; exit 1; }; done + @if [ "$(PROVIDER)" = "watsonx" ]; then \ + [ -n "$${WX_API_KEY:-}" ] && [ -n "$${WX_PROJECT_ID:-}" ] || { echo "ERROR: PROVIDER=watsonx needs WX_API_KEY/WX_PROJECT_ID (or run PROVIDER=ollama)"; exit 1; }; \ + fi + +host-setup: ## Ensure cluster internet egress works (MSS-clamp on hotspot/VPN networks) + KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) bash scripts/host-setup.sh + +ifeq ($(PROVIDER),watsonx) +ollama-up: ## watsonx: deploy in-cluster Ollama with the reasoning model (staged from host) + @if $(CONTAINER_RUNTIME) exec $(KIND_NODE) ctr -n k8s.io images list -q 2>/dev/null | grep -q "$(OLLAMA_IMAGE)"; then \ + echo "[*] $(OLLAMA_IMAGE) already present in node"; \ + else \ + $(CONTAINER_RUNTIME) image inspect $(OLLAMA_IMAGE) >/dev/null 2>&1 || $(CONTAINER_RUNTIME) pull $(OLLAMA_IMAGE); \ + kind load docker-image $(OLLAMA_IMAGE) --name $(KIND_CLUSTER_NAME); \ + $(CONTAINER_RUNTIME) exec $(KIND_NODE) ctr -n k8s.io images tag localhost/$(OLLAMA_IMAGE) docker.io/library/$(OLLAMA_IMAGE) >/dev/null 2>&1 || true; \ + fi + OLLAMA_MODEL=$(AGENT_OLLAMA_MODEL) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) bash scripts/stage-ollama-model.sh $(AGENT_OLLAMA_MODEL) + kubectl apply -f k8s/ollama.yaml + kubectl -n $(NS) rollout status deploy/ollama --timeout=180s + @echo "[*] warming $(AGENT_OLLAMA_MODEL) (loads into memory; KEEP_ALIVE keeps it resident) ..." + -kubectl -n $(NS) exec deploy/ollama -- ollama run $(AGENT_OLLAMA_MODEL) "hi" >/dev/null 2>&1 +else +ollama-up: ## ollama: use the host Ollama (Metal/GPU) for reasoning + reflection — no in-cluster Ollama + @echo "[*] PROVIDER=ollama: reasoning + reflection use the HOST Ollama at $(HOST_OLLAMA_URL)" + @echo " reasoning model=$(AGENT_OLLAMA_MODEL) reflection model=$(SPARC_MODEL)" + @echo " (ALTK reflects free-form on ollama; a CPU-only in-cluster Ollama is too slow, so we use the host.)" + @echo "[*] checking the cluster can reach the host Ollama ..." + @kubectl run sparc-ollama-probe-$$$$ --rm -i --restart=Never --image=curlimages/curl:8.10.1 --quiet -- \ + -s -m 10 -o /dev/null -w 'host Ollama %{http_code}\n' $(HOST_OLLAMA_URL)/api/tags 2>/dev/null \ + || echo " WARN: could not reach $(HOST_OLLAMA_URL) from the cluster. Ensure host Ollama runs (OLLAMA_HOST=0.0.0.0) and HOST_OLLAMA_URL is correct (Rancher/colima: http://192.168.5.2:11434)." +endif + +build-images: ## Build the agent, MCP server, and SPARC service images + @echo "[*] building $(SPARC_IMAGE)"; $(CONTAINER_RUNTIME) build -t $(SPARC_IMAGE) ../../sparc-service + @echo "[*] building $(AGENT_IMAGE)"; $(CONTAINER_RUNTIME) build -f finance-agent/Dockerfile -t $(AGENT_IMAGE) . + @echo "[*] building $(MCP_IMAGE)"; $(CONTAINER_RUNTIME) build -f finance-mcp/Dockerfile -t $(MCP_IMAGE) . + +build-authbridge: ## Build the authbridge image WITH the sparc plugin (operator's tag) + @echo "[*] building $(AUTHBRIDGE_IMAGE) (includes sparc plugin)" + cd ../.. && $(CONTAINER_RUNTIME) build -f cmd/authbridge-proxy/Dockerfile -t $(AUTHBRIDGE_IMAGE) . + +load-images: build-images build-authbridge ## Load all images into kind + @for img in $(AGENT_IMAGE) $(MCP_IMAGE) $(SPARC_IMAGE) $(AUTHBRIDGE_IMAGE); do \ + echo "[*] kind load $$img"; kind load docker-image "$$img" --name $(KIND_CLUSTER_NAME); \ + $(CONTAINER_RUNTIME) exec $(KIND_NODE) ctr -n k8s.io images tag localhost/$$img docker.io/library/$$img >/dev/null 2>&1 || true; \ + done + +sparc-config: ## Create the SPARC service ConfigMap (+ watsonx secret) for $(PROVIDER) + @if [ "$(PROVIDER)" = "watsonx" ]; then \ + kubectl -n $(PLATFORM_NS) create configmap sparc-service-config \ + --from-literal=SPARC_LLM_PROVIDER=watsonx --from-literal=SPARC_MODEL=$(SPARC_MODEL) \ + --from-literal=SPARC_TRACK=fast_track --from-literal=PORT=8090 \ + --dry-run=client -o yaml | kubectl apply -f - ; \ + kubectl -n $(PLATFORM_NS) create secret generic sparc-watsonx \ + --from-literal=WX_API_KEY="$${WX_API_KEY}" --from-literal=WX_PROJECT_ID="$${WX_PROJECT_ID}" \ + --from-literal=WX_URL="$${WX_URL:-https://us-south.ml.cloud.ibm.com}" \ + --dry-run=client -o yaml | kubectl apply -f - ; \ + else \ + kubectl -n $(PLATFORM_NS) create configmap sparc-service-config \ + --from-literal=SPARC_LLM_PROVIDER=ollama --from-literal=SPARC_MODEL=$(SPARC_MODEL) \ + --from-literal=SPARC_TRACK=fast_track --from-literal=PORT=8090 \ + --from-literal=OLLAMA_BASE_URL=$(SPARC_OLLAMA_URL) \ + --dry-run=client -o yaml | kubectl apply -f - ; \ + kubectl -n $(PLATFORM_NS) create secret generic sparc-watsonx --dry-run=client -o yaml | kubectl apply -f - ; \ + fi + @kubectl -n $(PLATFORM_NS) rollout restart deploy/sparc-service >/dev/null 2>&1 || true + +deploy: sparc-config ## Deploy the SPARC service, finance MCP server, and agent + kubectl apply -f k8s/sparc-service.yaml + kubectl apply -f k8s/finance-mcp.yaml + kubectl apply -f k8s/agent.yaml + @# Point the agent's reasoning LLM at the provider's Ollama (in-cluster for + @# watsonx, host for ollama). This rolls the agent; the operator then resets + @# its authbridge ConfigMap, which patch-config re-applies next. + kubectl -n $(NS) set env deploy/$(AGENT) -c agent OLLAMA_URL=$(AGENT_OLLAMA_URL) OLLAMA_MODEL=$(AGENT_OLLAMA_MODEL) >/dev/null + @# Pick up freshly-loaded images (apply is a no-op when only :latest content changed). + -kubectl -n $(NS) rollout restart deploy/finance-mcp >/dev/null 2>&1 + -kubectl -n $(NS) rollout restart deploy/$(AGENT) >/dev/null 2>&1 + $(MAKE) wait-pods + +wait-pods: ## Wait for service/MCP/agent + the operator-rendered authbridge ConfigMap + kubectl -n $(PLATFORM_NS) rollout status deploy/sparc-service --timeout=180s + kubectl -n $(NS) rollout status deploy/finance-mcp --timeout=120s + kubectl -n $(NS) rollout status deploy/$(AGENT) --timeout=180s + @for i in $$(seq 1 60); do kubectl -n $(NS) get cm authbridge-config-$(AGENT) >/dev/null 2>&1 && break; sleep 2; done + @kubectl -n $(NS) get cm authbridge-config-$(AGENT) >/dev/null 2>&1 || { echo "ERROR: operator didn't create authbridge-config-$(AGENT)"; exit 1; } + +patch-config: ## Enable the parsers + sparc in the agent's authbridge pipeline (hot-reload) + SPARC_TIMEOUT_MS=$(SPARC_TIMEOUT_MS) bash scripts/patch-sparc-config.sh $(NS) $(AGENT) + +setup-keycloak: ## Create the audience scope, ROPC client, and alice so inbound auth passes + @kubectl -n keycloak port-forward svc/keycloak-service 18081:8080 >/tmp/finance-kc-setup-pf.log 2>&1 & \ + KCPF=$$!; sleep 4; \ + NAMESPACE=$(NS) AGENT_SA=$(AGENT) KEYCLOAK_URL=http://localhost:18081 \ + uv run --with python-keycloak python scripts/setup_keycloak_finance.py; rc=$$?; \ + kill $$KCPF 2>/dev/null || true; exit $$rc + +drive: ## Drive the two-turn scenario through real auth and print SPARC verdicts + TURN_TIMEOUT=$(TURN_TIMEOUT) bash scripts/drive-demo.sh $(NS) $(AGENT) + +demo: preflight host-setup load-images ollama-up deploy patch-config setup-keycloak drive ## Full end-to-end demo + @echo + @echo "Demo complete. UI: http://kagenti-ui.localtest.me:8080 | forensic: make show-result" + +build-abctl: ## Build abctl into /tmp (for show-result) + cd ../../cmd/abctl && go build -o /tmp/abctl-finance-sparc-demo . + +show-result: build-abctl ## Launch abctl against the agent's session API + bash scripts/launch-abctl.sh $(NS) $(AGENT) + +undeploy: ## Remove demo resources (kagenti install untouched) + -kubectl -n $(NS) delete -f k8s/agent.yaml --ignore-not-found + -kubectl -n $(NS) delete -f k8s/finance-mcp.yaml --ignore-not-found + -kubectl -n $(NS) delete -f k8s/ollama.yaml --ignore-not-found + -kubectl delete -f k8s/sparc-service.yaml --ignore-not-found + -kubectl -n $(PLATFORM_NS) delete configmap sparc-service-config --ignore-not-found + -kubectl -n $(PLATFORM_NS) delete secret sparc-watsonx --ignore-not-found + -kubectl -n $(NS) delete configmap authbridge-config-$(AGENT) --ignore-not-found + -kubectl -n $(NS) delete secret -l kagenti.io/client-name=$(AGENT) --ignore-not-found + @echo "[*] demo resources removed." + +logs-agent: ## Tail agent + authbridge sidecar logs + @echo "--- agent ---"; kubectl -n $(NS) logs deploy/$(AGENT) -c agent --tail=80 + @echo "--- authbridge ---"; kubectl -n $(NS) logs deploy/$(AGENT) -c authbridge-proxy --tail=80 + +logs-sparc: ## Tail the SPARC reflection service logs + kubectl -n $(PLATFORM_NS) logs deploy/sparc-service --tail=80 + +logs-ollama: ## Tail the in-cluster Ollama logs + kubectl -n $(NS) logs deploy/ollama --tail=80 diff --git a/authbridge/demos/finance-sparc/README.md b/authbridge/demos/finance-sparc/README.md new file mode 100644 index 000000000..c2e20a558 --- /dev/null +++ b/authbridge/demos/finance-sparc/README.md @@ -0,0 +1,183 @@ +# finance-sparc demo — SPARC pre-tool reflection on a regular agent + +This demo shows the AuthBridge [`sparc` plugin](../../docs/sparc-plugin.md) catching a +hallucinated tool argument before it executes. SPARC's clarification is returned to the agent +as an MCP tool result, so the agent asks the user for the missing detail and then runs the +corrected call. + +The agent is an ordinary kagenti agent: it discovers its tools via MCP `tools/list`, reasons +with an Ollama model, and acts through an MCP finance backend. SPARC only sees what the agent +itself produces — the full conversation (including the system prompt), the discovered tool +specs in OpenAI format, and the proposed tool call. Those are collected by the parsers from the +agent's own LLM call; nothing is hardcoded or passed to SPARC out of band. + +## The scenario (two turns) + +1. "Refund my duplicate $450 subscription charge from last week." + The user gives no transaction id and there is no search tool, so the model invents a value + (e.g. `"$450"`) for the required `transaction_id` and calls `issue_refund`. SPARC + (`fast_track`) finds the argument ungrounded and rejects it; the plugin returns a + clarification as the tool result, and the agent relays it and asks for the exact id. +2. "The transaction id is TX4827. Please proceed." + Now the id is grounded in the conversation, SPARC approves, and the refund is issued. + +Example run (watsonx reflection): + +``` +Turn 1 agent> This tool call was not executed because it could not be verified... ask for the exact transaction_id +Turn 2 agent> Your transaction with ID TX4827 has been successfully refunded... + +SPARC modify/reflected tool=issue_refund score=0.00 # Turn 1: ungrounded "$450" → reject + clarify +SPARC allow/grounded tool=issue_refund score=1.00 # Turn 2: grounded TX4827 → approve → refund +``` + +## Prerequisites + +- A **kagenti kind cluster** with the AuthBridge operator. Core + SPIRE is enough: + ```bash + # from the kagenti repo: + scripts/kind/setup-kagenti.sh --with-spire + ``` +- `kubectl`, `kind`, `docker` (or `podman`), `python3`, `uv`. +- **Ollama models.** What you need depends on the reflection provider: + - **`make demo` (watsonx reflection):** just the agent's reasoning model in your host + Ollama store — `ollama pull llama3.2:1b`. The demo stages it into an *in-cluster* Ollama + (no host networking needed for this path). + - **`make demo PROVIDER=ollama` (local reflection):** a **running host Ollama** reachable + from the cluster, with a reasoning model and a **capable** reflection model: + ```bash + ollama pull llama3.2:3b # reasoning (clean tool calls) + ollama pull gemma4:e4b # reflection — needs a capable instruct model; 1b/3b + # hallucinate ALTK's function-selection metric. Override + # with SPARC_MODEL=. + ``` + Why the host Ollama for this path: ALTK reflects on ollama via **free-form + generate-then-validate** (it disables guided decoding for ollama), which is fast on a + Metal/GPU host but ~140s/call on a CPU-only in-cluster node. See [Provider notes](#provider-notes). + +## Run it + +```bash +cd authbridge/demos/finance-sparc + +# Option A — watsonx as the reflection LLM (best quality, fastest reflection): +set -a; . /path/to/your/.env; set +a # provides WX_API_KEY / WX_PROJECT_ID +make demo + +# Option B — fully local, no credentials (Ollama reflection on your host Ollama): +make demo PROVIDER=ollama +# Rancher Desktop / colima: the host Ollama isn't at host.docker.internal, so: +# make demo PROVIDER=ollama HOST_OLLAMA_URL=http://192.168.5.2:11434 +``` + +`make demo` is one-shot and idempotent. It: checks cluster egress (a no-op on healthy +networks), builds + loads images (including an authbridge image carrying the `sparc` plugin), +stages + warms the in-cluster Ollama model(s), deploys the SPARC service + finance MCP server + +agent, enables the pipeline, configures Keycloak, and drives the two-turn scenario **through +real inbound auth** (no jwt bypass), printing SPARC's verdicts. + +## How to read the output + +- **The two turns** print as `user>` / `agent>`. Turn 1's `agent>` is SPARC's clarification + (relayed by the agent); Turn 2's `agent>` is the successful refund. +- **The verdict summary** at the end (`SPARC / tool=… score=…`) is read straight + from the agent's AuthBridge session API: + - `modify/reflected` — SPARC rejected an ungrounded call and the clarification was returned. + - `allow/grounded` — SPARC approved a grounded call; the tool ran. + - `score` is SPARC's confidence (0=worst..1=best); shown only when the model returns one. +- **Full forensic view** — the per-plugin pipeline timeline and the structured SPARC event: + ```bash + make show-result # abctl TUI against the agent's session API + make logs-sparc # the SPARC reflection service logs + make logs-agent # agent + authbridge sidecar logs + ``` +- **Re-run just the conversation** (cluster already set up): `make drive`. +- **Tear down the demo** (kagenti install untouched): `make undeploy`. + +## How it works (and why it's generic) + +- The agent's LLM call goes through the AuthBridge forward proxy, where **`inference-parser`** + captures the messages (incl. the system prompt) and the OpenAI tool specs, and **`mcp-parser`** + captures the outbound `tools/call`. The **`sparc`** plugin correlates those into SPARC's three + inputs — no per-agent wiring. +- On a reject, the plugin returns SPARC's clarification as a JSON-RPC **MCP tool result** marked + `_meta.sparc.reflected`. The agent surfaces that to the user and stops re-trying the call + (honoring its system instruction to relay "could not verify / clarify" results) — exactly how + a well-behaved agent should treat a tool that asks for clarification. +- Reflection runs **out-of-band** (the plugin calls the SPARC service directly), so it is *not* + bound by the proxy's 30s upstream timeout. `make demo` sets a generous reflection `timeout_ms` + automatically for `PROVIDER=ollama`. + +## What's in here + +| Path | What | +|---|---| +| `finance-agent/` | A regular A2A agent: Ollama reasoning, MCP `tools/list` discovery, MCP tool execution. | +| `finance-mcp/` | MCP server: `get_transaction`, `lookup_customer`, `issue_refund`, `get_invoice`, `list_currencies` over a small multi-record dataset (real id: `TX4827`). | +| `k8s/sparc-patch.yaml` | Pipeline additions: `a2a-parser` inbound; `inference-parser` + `mcp-parser` + `sparc` outbound (mcp mode, fast_track, reflect, `skip_tools: [list_currencies]`). | +| `k8s/{agent,finance-mcp,sparc-service}.yaml` | Workload manifests. | +| `k8s/ollama.yaml` | In-cluster Ollama for the agent's reasoning model on the **watsonx** path; model staged onto the node — no host networking. (The `PROVIDER=ollama` path uses your host Ollama instead.) | +| `scripts/host-setup.sh` | Generic egress check (no-op on healthy networks; portable MSS clamp only if the link blackholes PMTUD). | +| `scripts/stage-ollama-model.sh` | Copies a model from the host Ollama store onto the kind node (local, no pull). | +| `scripts/setup_keycloak_finance.py` | Audience scope (realm default) + ROPC client + `alice` so inbound auth passes (scripted and UI). | +| `scripts/drive-demo.sh` | Gets a real user token and runs the two-turn scenario, printing SPARC verdicts. | +| `scripts/patch-sparc-config.sh` | Merges the pipeline patch into the operator's authbridge ConfigMap (hot-reload); substitutes the provider-tuned reflection timeout. | +| `Makefile` | `demo` (one-shot), `drive`, `show-result`, `undeploy`, `logs-*`. | + +## Provider notes + +- **watsonx (default)** — reflection on watsonx (`mistral-large-2512`, ~5s); the agent reasons + with an in-cluster Ollama (`llama3.2:1b`). Fastest, highest-quality, no host Ollama needed. + Verified: Turn 1 `modify/reflected score=0.00`, Turn 2 `allow/grounded score=1.00`. +- **ollama (`PROVIDER=ollama`)** — both reasoning and reflection run on your **host** Ollama. + Verified: Turn 1 `modify/reflected score=0.50`, Turn 2 `allow/grounded score=1.00`. + Two things make the host the right place for this path: + - **Reflection latency.** ALTK reflects on ollama via free-form generate-then-validate + (guided decoding is disabled for ollama in ALTK). That's ~140s/call on a CPU-only in-cluster + node (over the agent's 240s deadline → fails open), but a few seconds on a Metal/GPU host. + - **Reflection quality.** The function-selection metric needs a **capable** model. `llama3.2` + 1b/3b *hallucinate* it (they invent unrelated tools and wrongly reject grounded calls); + a larger instruct model (e.g. `gemma4:e4b`, the verified default) judges correctly. Override + with `SPARC_MODEL=`. + - The agent's reasoning model on this path is `llama3.2:3b` (a clean tool-caller; 1b tends to + emit malformed tool calls). Override with `AGENT_OLLAMA_MODEL=...`. + - `make demo PROVIDER=ollama` raises the reflection `timeout_ms` (200s) and per-turn timeout + (600s) to absorb first-call model loads. +- Small models are non-deterministic. The agent stops and relays on SPARC's clarification, so a + run won't loop; if the model ever declines to fabricate an id on Turn 1, re-run `make drive`. + +## Troubleshooting + +- **Image pulls fail with DNS errors** (`server misbehaving`, `could not resolve host`): your + container runtime's VM resolver went stale after a network change (common with Docker + Desktop / Rancher Desktop / colima after switching Wi-Fi or VPN). Restart the runtime, or + point its VM and the kind node at a public resolver (`8.8.8.8`). This is a host-runtime issue, + not a demo or kagenti one — the demo scripts contain no machine-specific networking. +- **`authbridge-config-` reverted to defaults**: the operator rewrites it whenever the + agent pod rolls. `make demo` (and `make patch-config`) re-apply the SPARC pipeline and wait + for the live config SHA to converge, so just re-run the relevant target. +- **Agent pods stuck `ContainerCreating` on a missing `kagenti-keycloak-client-credentials-*` + secret** (operator log: *"waiting for KEYCLOAK_URL/KEYCLOAK_REALM in authbridge-config"* or + *"waiting for keycloak-admin-secret"*): a platform/operator-version skew — the operator + registers the agent's Keycloak client from resources a matching chart version provides. This + affects **every** agent, not just this demo. Provide them once per cluster: + ```bash + kubectl -n team1 create configmap authbridge-config \ + --from-literal=KEYCLOAK_URL=http://keycloak-service.keycloak:8080 \ + --from-literal=KEYCLOAK_REALM=kagenti --dry-run=client -o yaml | kubectl apply -f - + kubectl -n kagenti-system create secret generic keycloak-admin-secret \ + --from-literal=KEYCLOAK_ADMIN_USERNAME=admin \ + --from-literal=KEYCLOAK_ADMIN_PASSWORD=admin --dry-run=client -o yaml | kubectl apply -f - + ``` + A kagenti install whose chart and operator versions are aligned creates these automatically. +- **`PROVIDER=ollama`: `reflector_unavailable` / can't reach the host Ollama**: the cluster + must reach your host Ollama. Run it with `OLLAMA_HOST=0.0.0.0`, and set `HOST_OLLAMA_URL` + correctly (Docker Desktop: `http://host.docker.internal:11434`; Rancher Desktop / colima: + `http://192.168.5.2:11434`). `make demo PROVIDER=ollama` probes reachability and warns if it + fails. A `reject` on the grounded turn usually means too small a reflection model — set + `SPARC_MODEL` to a capable instruct model your host Ollama has pulled. + +## See also +- [`docs/sparc-plugin.md`](../../docs/sparc-plugin.md) — plugin reference (config, modes). +- [`sparc-service/README.md`](../../sparc-service/README.md) — the reflection service. +- [`demos/ibac/`](../ibac/README.md) — complementary intent control (SPARC verifies grounding; IBAC verifies intent). diff --git a/authbridge/demos/finance-sparc/finance-agent/Dockerfile b/authbridge/demos/finance-sparc/finance-agent/Dockerfile new file mode 100644 index 000000000..f92c3e41c --- /dev/null +++ b/authbridge/demos/finance-sparc/finance-agent/Dockerfile @@ -0,0 +1,12 @@ +# Multi-stage build for the finance-sparc demo agent (stdlib only). +FROM golang:1.24-alpine AS build +WORKDIR /src +COPY go.mod ./ +COPY finance-agent/main.go ./finance-agent/main.go +RUN CGO_ENABLED=0 go build -o /out/agent ./finance-agent + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/agent /agent +# Listens on $PORT (default 8080). The demo Pod sets PORT=8000 to leave +# 8080 for the operator-injected authbridge sidecar's reverse proxy. +ENTRYPOINT ["/agent"] diff --git a/authbridge/demos/finance-sparc/finance-agent/main.go b/authbridge/demos/finance-sparc/finance-agent/main.go new file mode 100644 index 000000000..33c5cc3ed --- /dev/null +++ b/authbridge/demos/finance-sparc/finance-agent/main.go @@ -0,0 +1,471 @@ +// Finance assistant agent for the SPARC demo. +// +// A regular kagenti A2A agent (JSON-RPC over HTTP at /). It reasons with a +// local Ollama model and acts through a finance MCP server. It DISCOVERS its +// tools at runtime via MCP `tools/list` — nothing about the tools is hardcoded +// — so it behaves like any normal agent. There is no demo-specific steering or +// side-channel: SPARC sees only what the agent genuinely sends to its LLM (full +// conversation incl. system prompt, the discovered tool specs) and the tool +// call it makes. +// +// The agent is configured to be proactive (complete the user's request with the +// available tools). When a user refers to a charge descriptively — without a +// transaction id — the model fabricates a precise id to call a tool. The +// AuthBridge `sparc` plugin reflects on that call, finds the argument +// ungrounded, and returns a clarification as the MCP tool result; the agent +// relays it and asks the user for the exact id. Once supplied, SPARC approves +// and the refund proceeds. +package main + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" +) + +// --- OpenAI-compatible chat types (Ollama /v1/chat/completions) --- + +type ChatRequest struct { + Model string `json:"model"` + Messages []ChatMessage `json:"messages"` + Tools []Tool `json:"tools,omitempty"` + ToolChoice string `json:"tool_choice,omitempty"` + Temperature float64 `json:"temperature,omitempty"` +} + +type ChatMessage struct { + Role string `json:"role"` + Content string `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` +} + +type ToolCall struct { + ID string `json:"id"` + Type string `json:"type"` + Function FunctionCall `json:"function"` +} + +type FunctionCall struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +type Tool struct { + Type string `json:"type"` + Function ToolFunction `json:"function"` +} + +type ToolFunction struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters map[string]any `json:"parameters"` +} + +type ChatResponse struct { + Choices []ChatChoice `json:"choices"` +} + +type ChatChoice struct { + Message ChatMessage `json:"message"` +} + +// --- Outbound HTTP via the authbridge forward proxy --- + +var proxiedClient *http.Client + +func buildProxiedClient() *http.Client { + proxyEnv := os.Getenv("HTTP_PROXY") + if proxyEnv == "" { + log.Printf("[Agent] HTTP_PROXY unset — outbound HTTP will be direct (authbridge will not see it)") + return &http.Client{Timeout: 240 * time.Second} + } + u, err := url.Parse(proxyEnv) + if err != nil { + log.Printf("[Agent] HTTP_PROXY=%q invalid (%v) — direct", proxyEnv, err) + return &http.Client{Timeout: 240 * time.Second} + } + log.Printf("[Agent] outbound HTTP via proxy: %s", u) + return &http.Client{Timeout: 240 * time.Second, Transport: &http.Transport{Proxy: http.ProxyURL(u)}} +} + +// --- MCP (tool discovery + execution) --- + +func mcpURL() string { return envOr("FINANCE_MCP_URL", "http://localhost:8888") } + +func mcpCall(method string, params any) (map[string]any, error) { + rpc := map[string]any{"jsonrpc": "2.0", "id": fmt.Sprintf("%d", time.Now().UnixNano()), "method": method} + if params != nil { + rpc["params"] = params + } + body, _ := json.Marshal(rpc) + req, err := http.NewRequest(http.MethodPost, mcpURL()+"/mcp", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := proxiedClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + raw, _ := io.ReadAll(resp.Body) + var env map[string]any + if err := json.Unmarshal(raw, &env); err != nil { + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(raw)) + } + return env, nil +} + +var ( + toolsMu sync.Mutex + toolsCache []Tool +) + +// discoverTools fetches the MCP tool inventory via tools/list and maps it to +// OpenAI tool specs for the LLM. The result is cached only on success, so a +// transient failure (MCP not ready yet, a flaky call) is retried on the next +// turn rather than poisoning the agent with an empty tool list for its lifetime. +func discoverTools() []Tool { + toolsMu.Lock() + defer toolsMu.Unlock() + if len(toolsCache) > 0 { + return toolsCache + } + env, err := mcpCall("tools/list", nil) + if err != nil { + log.Printf("[Agent] tools/list failed (will retry): %v", err) + return nil + } + result, _ := env["result"].(map[string]any) + list, _ := result["tools"].([]any) + var discovered []Tool + for _, t := range list { + tm, _ := t.(map[string]any) + name, _ := tm["name"].(string) + if name == "" { + continue + } + desc, _ := tm["description"].(string) + params, _ := tm["inputSchema"].(map[string]any) + discovered = append(discovered, Tool{Type: "function", Function: ToolFunction{Name: name, Description: desc, Parameters: params}}) + } + if len(discovered) == 0 { + log.Printf("[Agent] tools/list returned no tools (will retry)") + return nil + } + toolsCache = discovered + log.Printf("[Agent] discovered %d tools via MCP tools/list", len(toolsCache)) + return toolsCache +} + +// execMCPTool runs a tool via tools/call and returns its text result (real data, +// or the SPARC clarification injected in place of an ungrounded call). The second +// return value is true when the result is a SPARC clarification (MCP `_meta.sparc. +// reflected`) — the agent surfaces that to the user rather than retrying the call. +func execMCPTool(name string, args map[string]any) (string, bool) { + env, err := mcpCall("tools/call", map[string]any{"name": name, "arguments": args}) + if err != nil { + return fmt.Sprintf("error calling tool: %v", err), false + } + if e, ok := env["error"].(map[string]any); ok { + return fmt.Sprintf("tool error: %v", e["message"]), false + } + result, _ := env["result"].(map[string]any) + reflected := false + if meta, ok := result["_meta"].(map[string]any); ok { + if sp, ok := meta["sparc"].(map[string]any); ok { + reflected, _ = sp["reflected"].(bool) + } + } + content, _ := result["content"].([]any) + var parts []string + for _, c := range content { + cm, _ := c.(map[string]any) + if txt, _ := cm["text"].(string); txt != "" { + parts = append(parts, txt) + } + } + if len(parts) == 0 { + return toJSONStr(result), reflected + } + return strings.Join(parts, "\n"), reflected +} + +func toJSONStr(v any) string { b, _ := json.Marshal(v); return string(b) } + +// --- Ollama interaction --- + +func callOllama(messages []ChatMessage, tools []Tool) (*ChatResponse, error) { + ollamaURL := envOr("OLLAMA_URL", "http://localhost:11434") + chatReq := ChatRequest{Model: envOr("OLLAMA_MODEL", "llama3.2:3b"), Messages: messages, Temperature: 0.1} + if len(tools) > 0 { + chatReq.Tools = tools + chatReq.ToolChoice = "auto" + } + reqBody, _ := json.Marshal(chatReq) + // Use the proxied client: it routes through the authbridge forward proxy + // (so inference-parser captures the messages + tool specs SPARC needs) and + // carries the explicit timeout — http.Post's default client has neither. + req, err := http.NewRequest(http.MethodPost, ollamaURL+"/v1/chat/completions", bytes.NewReader(reqBody)) + if err != nil { + return nil, fmt.Errorf("build ollama request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := proxiedClient.Do(req) + if err != nil { + return nil, fmt.Errorf("call ollama: %w", err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("ollama returned %d: %s", resp.StatusCode, string(body)) + } + var cr ChatResponse + if err := json.Unmarshal(body, &cr); err != nil { + return nil, fmt.Errorf("unmarshal ollama response: %w", err) + } + return &cr, nil +} + +func envOr(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +// --- Agent loop --- +// +// A normal proactive finance-assistant prompt. No id-guessing instructions and +// no ground-truth leakage — the hallucination, when it happens, comes from the +// model trying to act on a request that's missing a required argument. +const systemPrompt = "You are a finance operations assistant. Use the available tools to fulfil the user's " + + "request. To refund a charge you must call issue_refund with the transaction's id. Take action with the " + + "tools rather than replying in prose. If a tool result tells you a value could not be verified or asks you " + + "to clarify, relay that to the user and ask for the exact missing detail." + +func runAgent(query string) (string, error) { + tools := discoverTools() + messages := []ChatMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: query}, + } + lastToolResult := "" + for i := 0; i < 8; i++ { + resp, err := callOllama(messages, tools) + if err != nil { + return "", err + } + if len(resp.Choices) == 0 { + return "", fmt.Errorf("no choices in ollama response") + } + msg := resp.Choices[0].Message + if len(msg.ToolCalls) == 0 { + return msg.Content, nil + } + messages = append(messages, msg) + for _, tc := range msg.ToolCalls { + log.Printf("[Agent] tool call: %s(%s)", tc.Function.Name, tc.Function.Arguments) + var args map[string]any + if err := json.Unmarshal([]byte(tc.Function.Arguments), &args); err != nil { + args = map[string]any{} + } + result, reflected := execMCPTool(tc.Function.Name, args) + lastToolResult = result + log.Printf("[Agent] tool result (%s, reflected=%t): %.200s", tc.Function.Name, reflected, result) + messages = append(messages, ChatMessage{Role: "tool", Content: result, ToolCallID: tc.ID}) + if reflected { + // SPARC returned a clarification in place of the tool result + // (an ungrounded/unverifiable argument). Relay it to the user + // and stop — retrying would hit the same rejection and only + // grow the prompt. This is the agent honoring its system + // instruction to surface "could not verify / clarify" results. + return result, nil + } + } + } + // The model kept calling tools without concluding (common with small models + // when a tool keeps returning a clarification). Surface the last tool result + // to the user rather than a bare error — it carries the actionable guidance. + if lastToolResult != "" { + return lastToolResult, nil + } + return "", fmt.Errorf("tool-calling loop exceeded max iterations") +} + +// --- A2A (JSON-RPC 2.0) endpoint --- + +type a2aRequest struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id"` + Method string `json:"method"` + Params struct { + Message struct { + Role string `json:"role"` + Parts []part `json:"parts"` + ContextID string `json:"contextId,omitempty"` + } `json:"message"` + } `json:"params"` +} + +type part struct { + Kind string `json:"kind"` + Text string `json:"text,omitempty"` +} + +type a2aResponse struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id"` + Result *a2aTask `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +type a2aTask struct { + ID string `json:"id"` + ContextID string `json:"contextId,omitempty"` + Kind string `json:"kind"` + Status a2aStatus `json:"status"` + Artifacts []a2aArtifact `json:"artifacts,omitempty"` +} + +type a2aStatus struct { + State string `json:"state"` + Message *a2aMessage `json:"message,omitempty"` +} + +type a2aMessage struct { + Role string `json:"role"` + Parts []part `json:"parts"` +} + +type a2aArtifact struct { + ArtifactID string `json:"artifactId"` + Name string `json:"name,omitempty"` + Parts []part `json:"parts"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +func handleAgentCard(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + publicURL := envOr("AGENT_PUBLIC_URL", "http://finance-agent.team1.svc.cluster.local:8080/") + card := map[string]any{ + "name": "Finance Assistant", + "description": "Finance operations agent: transaction lookups and refunds via an MCP finance backend. AuthBridge's SPARC plugin reflects on each tool call and blocks ungrounded/hallucinated arguments, transparently asking for clarification.", + "protocolVersion": "0.3.0", + "version": "0.0.1", + "url": publicURL, + "preferredTransport": "JSONRPC", + "defaultInputModes": []string{"text"}, + "defaultOutputModes": []string{"text"}, + "capabilities": map[string]any{"streaming": false}, + "skills": []map[string]any{{ + "id": "refund", "name": "Refund a transaction", + "description": "Looks up and refunds a transaction; SPARC verifies the transaction id is grounded before any refund runs.", + "tags": []string{"demo", "finance", "sparc"}, + "examples": []string{"Refund my duplicate $450 subscription charge from last week."}, + }}, + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(card); err != nil { + log.Printf("[Agent] encode agent card: %v", err) + } +} + +func handleA2A(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "read body", http.StatusBadRequest) + return + } + var req a2aRequest + if err := json.Unmarshal(body, &req); err != nil { + writeRPCError(w, nil, -32700, "parse error: "+err.Error()) + return + } + if req.Method != "message/send" { + writeRPCError(w, req.ID, -32601, "method not found: "+req.Method) + return + } + var query string + for _, p := range req.Params.Message.Parts { + if p.Kind == "text" && p.Text != "" { + query = p.Text + break + } + } + if query == "" { + writeRPCError(w, req.ID, -32602, "no text part in message") + return + } + sessionID := req.Params.Message.ContextID + if sessionID == "" { + sessionID = newUUID() + } + log.Printf("[Agent] A2A query (session=%s): %s", sessionID, query) + result, err := runAgent(query) + if err != nil { + writeRPCError(w, req.ID, -32603, err.Error()) + return + } + writeRPCSuccess(w, req.ID, sessionID, result) +} + +func writeRPCSuccess(w http.ResponseWriter, id any, sessionID, text string) { + parts := []part{{Kind: "text", Text: text}} + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(a2aResponse{ + JSONRPC: "2.0", ID: id, + Result: &a2aTask{ + ID: newUUID(), ContextID: sessionID, Kind: "task", + Status: a2aStatus{State: "completed", Message: &a2aMessage{Role: "agent", Parts: parts}}, + Artifacts: []a2aArtifact{{ArtifactID: newUUID(), Name: "reply", Parts: parts}}, + }, + }) +} + +func newUUID() string { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return fmt.Sprintf("%d", time.Now().UnixNano()) + } + return hex.EncodeToString(b[:]) +} + +func writeRPCError(w http.ResponseWriter, id any, code int, message string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(a2aResponse{JSONRPC: "2.0", ID: id, Error: &rpcError{Code: code, Message: message}}) +} + +func main() { + proxiedClient = buildProxiedClient() + http.HandleFunc("/", handleA2A) + http.HandleFunc("/.well-known/agent-card.json", handleAgentCard) + port := envOr("PORT", "8080") + log.Printf("[Agent] Finance assistant starting on :%s (A2A at /)", port) + if err := http.ListenAndServe(":"+port, nil); err != nil { //nolint:gosec // demo + log.Fatalf("server: %v", err) + } +} diff --git a/authbridge/demos/finance-sparc/finance-mcp/Dockerfile b/authbridge/demos/finance-sparc/finance-mcp/Dockerfile new file mode 100644 index 000000000..1d0db94b8 --- /dev/null +++ b/authbridge/demos/finance-sparc/finance-mcp/Dockerfile @@ -0,0 +1,11 @@ +# Multi-stage build for the finance-sparc demo MCP server (stdlib only). +FROM golang:1.24-alpine AS build +WORKDIR /src +COPY go.mod ./ +COPY finance-mcp/main.go ./finance-mcp/main.go +RUN CGO_ENABLED=0 go build -o /out/finance-mcp ./finance-mcp + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/finance-mcp /finance-mcp +EXPOSE 8888 +ENTRYPOINT ["/finance-mcp"] diff --git a/authbridge/demos/finance-sparc/finance-mcp/main.go b/authbridge/demos/finance-sparc/finance-mcp/main.go new file mode 100644 index 000000000..c40104d91 --- /dev/null +++ b/authbridge/demos/finance-sparc/finance-mcp/main.go @@ -0,0 +1,220 @@ +// Finance MCP server for the SPARC demo. +// +// Speaks MCP (Model Context Protocol) over HTTP — JSON-RPC 2.0 POSTs to /mcp. +// Serves a small but non-trivial finance dataset through five tools and +// advertises them via tools/list (so the agent discovers them at runtime like +// any normal kagenti agent — nothing is hardcoded on the agent side). +// +// Tools: +// +// get_transaction(transaction_id) → transaction details +// lookup_customer(customer_id) → customer details +// issue_refund(transaction_id[, reason]) → refund confirmation (id required) +// get_invoice(invoice_id) → invoice details +// list_currencies() → static reference data (demo of skip_tools) +// +// There is deliberately NO search/list-transactions tool: when a user refers to +// a charge descriptively (no id), the agent cannot look the id up — a proactive +// model will fabricate a precise transaction_id, which SPARC catches as +// ungrounded before any refund touches the wrong record. +package main + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "strings" +) + +type jsonRPCRequest struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params,omitempty"` +} + +type jsonRPCResponse struct { + JSONRPC string `json:"jsonrpc"` + ID any `json:"id"` + Result any `json:"result,omitempty"` + Error *jsonRPCError `json:"error,omitempty"` +} + +type jsonRPCError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type mcpContent struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` +} + +type mcpToolCallResult struct { + Content []mcpContent `json:"content"` + IsError bool `json:"isError,omitempty"` +} + +type mcpToolCallParams struct { + Name string `json:"name"` + Arguments map[string]any `json:"arguments,omitempty"` +} + +// --- In-memory finance dataset (several of each entity) --- + +var transactions = map[string]map[string]any{ + "TX4827": {"transaction_id": "TX4827", "amount": 450.00, "currency": "USD", "customer_id": "C921", "status": "settled", "description": "Annual subscription renewal"}, + "TX5310": {"transaction_id": "TX5310", "amount": 1200.00, "currency": "USD", "customer_id": "C742", "status": "settled", "description": "Hardware purchase"}, + "TX2981": {"transaction_id": "TX2981", "amount": 75.50, "currency": "EUR", "customer_id": "C921", "status": "pending", "description": "Bookstore order"}, + "TX6644": {"transaction_id": "TX6644", "amount": 99.00, "currency": "USD", "customer_id": "C305", "status": "settled", "description": "Monthly subscription"}, +} + +var customers = map[string]map[string]any{ + "C921": {"customer_id": "C921", "name": "Daniel Reed", "email": "daniel.reed@example.com", "tier": "gold"}, + "C742": {"customer_id": "C742", "name": "Maria Gomez", "email": "maria.gomez@example.com", "tier": "silver"}, + "C305": {"customer_id": "C305", "name": "Sven Olsen", "email": "sven.olsen@example.com", "tier": "bronze"}, +} + +var invoices = map[string]map[string]any{ + "INV-8834": {"invoice_id": "INV-8834", "vendor": "Acme Corp", "amount": 1280.00, "currency": "USD", "status": "due"}, + "INV-9102": {"invoice_id": "INV-9102", "vendor": "Globex", "amount": 540.00, "currency": "USD", "status": "paid"}, +} + +func toJSON(v any) string { b, _ := json.Marshal(v); return string(b) } + +func strProp(desc string) map[string]any { + return map[string]any{"type": "string", "description": desc} +} + +func fn(name, desc string, props map[string]any, required []string) map[string]any { + return map[string]any{ + "name": name, + "description": desc, + "parameters": map[string]any{"type": "object", "properties": props, "required": required}, + } +} + +// toolSpecs advertised via tools/list (OpenAI function-calling shape under +// inputSchema, the MCP convention the agent maps to its LLM tool list). +func toolSpecs() []map[string]any { + return []map[string]any{ + fn("get_transaction", "Fetch transaction details by exact transaction id.", + map[string]any{"transaction_id": strProp("Exact transaction identifier")}, []string{"transaction_id"}), + fn("lookup_customer", "Fetch customer details by exact customer id.", + map[string]any{"customer_id": strProp("Exact customer identifier")}, []string{"customer_id"}), + fn("issue_refund", "Issue a refund for a transaction. Requires the exact transaction_id.", + map[string]any{ + "transaction_id": strProp("Exact transaction identifier to refund"), + "reason": strProp("Optional reason for the refund"), + }, []string{"transaction_id"}), + fn("get_invoice", "Fetch an invoice by exact invoice id.", + map[string]any{"invoice_id": strProp("Exact invoice identifier")}, []string{"invoice_id"}), + fn("list_currencies", "List supported reference currencies (static data).", + map[string]any{}, []string{}), + } +} + +func callTool(name string, args map[string]any) mcpToolCallResult { + get := func(k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + notFound := func(kind, id string) mcpToolCallResult { + return mcpToolCallResult{Content: []mcpContent{{Type: "text", Text: fmt.Sprintf("%s %q not found.", kind, id)}}, IsError: true} + } + ok := func(v any) mcpToolCallResult { + return mcpToolCallResult{Content: []mcpContent{{Type: "text", Text: toJSON(v)}}} + } + switch name { + case "get_transaction": + if t, found := transactions[get("transaction_id")]; found { + return ok(t) + } + return notFound("transaction", get("transaction_id")) + case "lookup_customer": + if c, found := customers[get("customer_id")]; found { + return ok(c) + } + return notFound("customer", get("customer_id")) + case "issue_refund": + tid := get("transaction_id") + if _, found := transactions[tid]; !found { + return notFound("transaction", tid) + } + return ok(map[string]any{"status": "refund_issued", "transaction_id": tid, "reason": get("reason")}) + case "get_invoice": + if inv, found := invoices[get("invoice_id")]; found { + return ok(inv) + } + return notFound("invoice", get("invoice_id")) + case "list_currencies": + return ok([]string{"USD", "EUR", "GBP", "JPY"}) + default: + return mcpToolCallResult{Content: []mcpContent{{Type: "text", Text: "unknown tool: " + name}}, IsError: true} + } +} + +func writeError(w http.ResponseWriter, id any, code int, message string) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(jsonRPCResponse{JSONRPC: "2.0", ID: id, Error: &jsonRPCError{Code: code, Message: message}}) +} + +func writeResult(w http.ResponseWriter, id any, result any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(jsonRPCResponse{JSONRPC: "2.0", ID: id, Result: result}) +} + +func handleMCP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + writeError(w, nil, -32700, "read body: "+err.Error()) + return + } + var req jsonRPCRequest + if err := json.Unmarshal(body, &req); err != nil { + writeError(w, nil, -32700, "parse error: "+err.Error()) + return + } + slog.Info("mcp request", "method", req.Method, "id", req.ID) + switch req.Method { + case "initialize": + writeResult(w, req.ID, map[string]any{ + "protocolVersion": "2024-11-05", + "capabilities": map[string]any{"tools": map[string]any{}}, + "serverInfo": map[string]any{"name": "finance-mcp", "version": "0.1.0"}, + }) + case "tools/list": + tools := make([]map[string]any, 0) + for _, s := range toolSpecs() { + tools = append(tools, map[string]any{"name": s["name"], "description": s["description"], "inputSchema": s["parameters"]}) + } + writeResult(w, req.ID, map[string]any{"tools": tools}) + case "tools/call": + var params mcpToolCallParams + if len(req.Params) > 0 { + if err := json.Unmarshal(req.Params, ¶ms); err != nil { + writeError(w, req.ID, -32602, "params: "+err.Error()) + return + } + } + res := callTool(params.Name, params.Arguments) + slog.Info("tools/call", "tool", params.Name, "args", toJSON(params.Arguments), "isError", res.IsError) + writeResult(w, req.ID, res) + default: + writeError(w, req.ID, -32601, "method not found: "+req.Method) + } +} + +func main() { + http.HandleFunc("/mcp", handleMCP) + addr := ":8888" + slog.Info("finance MCP server starting", "addr", addr+"/mcp") + if err := http.ListenAndServe(addr, nil); err != nil { //nolint:gosec // demo + slog.Error("failed to start finance server", "error", err) + os.Exit(1) + } +} diff --git a/authbridge/demos/finance-sparc/go.mod b/authbridge/demos/finance-sparc/go.mod new file mode 100644 index 000000000..d4bebc7cc --- /dev/null +++ b/authbridge/demos/finance-sparc/go.mod @@ -0,0 +1,3 @@ +module github.com/kagenti/kagenti-extensions/authbridge/demos/finance-sparc + +go 1.24 diff --git a/authbridge/demos/finance-sparc/k8s/agent.yaml b/authbridge/demos/finance-sparc/k8s/agent.yaml new file mode 100644 index 000000000..59f62b567 --- /dev/null +++ b/authbridge/demos/finance-sparc/k8s/agent.yaml @@ -0,0 +1,121 @@ +# finance-sparc demo agent — operator-injected, kagenti-UI-discoverable. +# +# Required labels (same contract as demos/ibac/k8s/agent.yaml): +# kagenti.io/type: agent — UI lists agents by this label +# kagenti.io/inject: enabled — operator's webhook injects authbridge +# kagenti.io/spire: disabled — demo doesn't depend on SPIRE identity +# protocol.kagenti.io/a2a: "" — declares A2A for the chat flow +# +# The operator port-steals (containerPort 8000 → agent on 8001, PORT and +# HTTP_PROXY set so outbound flows through the injected sidecar's forward +# proxy). We do NOT ship a per-agent authbridge ConfigMap — the operator +# creates authbridge-config-finance-agent; scripts/patch-sparc-config.sh +# enables the parsers + sparc plugin after deploy. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: finance-agent + namespace: team1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: finance-agent + namespace: team1 + labels: + app.kubernetes.io/name: finance-agent + app.kubernetes.io/part-of: finance-sparc-demo + kagenti.io/type: agent + protocol.kagenti.io/a2a: "" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: finance-agent + template: + metadata: + labels: + app.kubernetes.io/name: finance-agent + app.kubernetes.io/part-of: finance-sparc-demo + kagenti.io/type: agent + kagenti.io/inject: enabled + kagenti.io/spire: disabled + protocol.kagenti.io/a2a: "" + spec: + serviceAccountName: finance-agent + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: agent + image: finance-agent:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8000 + name: http + env: + - name: PORT + value: "8000" + - name: OLLAMA_URL + value: "http://ollama.team1.svc.cluster.local:11434" + - name: OLLAMA_MODEL + value: "llama3.2:1b" + - name: FINANCE_MCP_URL + value: "http://finance-mcp.team1.svc.cluster.local:8888" + - name: AGENT_PUBLIC_URL + value: "http://finance-agent.team1.svc.cluster.local:8080/" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 50m + memory: 64Mi + # /shared is mounted by the operator-injected authbridge sidecar + # (it reads /shared/client-id.txt). The agent doesn't read it; + # the volume just has to exist on the Pod. + volumeMounts: + - name: shared-data + mountPath: /shared + volumes: + - name: shared-data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: finance-agent + namespace: team1 + labels: + app.kubernetes.io/name: finance-agent + app.kubernetes.io/part-of: finance-sparc-demo +spec: + selector: + app.kubernetes.io/name: finance-agent + ports: + - port: 8080 + targetPort: 8000 + protocol: TCP + name: http + type: ClusterIP +--- +# AgentRuntime triggers operator-managed Keycloak client registration +# (the Secret the authbridge sidecar mounts at /shared/). The operator +# auto-creates the AgentCard CR for the kagenti.io/type=agent Deployment. +apiVersion: agent.kagenti.dev/v1alpha1 +kind: AgentRuntime +metadata: + name: finance-agent + namespace: team1 +spec: + type: agent + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: finance-agent diff --git a/authbridge/demos/finance-sparc/k8s/finance-mcp.yaml b/authbridge/demos/finance-sparc/k8s/finance-mcp.yaml new file mode 100644 index 000000000..208a4474a --- /dev/null +++ b/authbridge/demos/finance-sparc/k8s/finance-mcp.yaml @@ -0,0 +1,63 @@ +# Finance MCP server — the tool backend the finance-agent calls. +# Plain in-cluster workload (no authbridge injection); the agent reaches +# it at http://finance-mcp.team1.svc.cluster.local:8888/mcp. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: finance-mcp + namespace: team1 + labels: + app.kubernetes.io/name: finance-mcp + app.kubernetes.io/part-of: finance-sparc-demo +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: finance-mcp + template: + metadata: + labels: + app.kubernetes.io/name: finance-mcp + app.kubernetes.io/part-of: finance-sparc-demo + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: finance-mcp + image: finance-mcp:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8888 + name: http + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + limits: + cpu: 200m + memory: 64Mi + requests: + cpu: 20m + memory: 16Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: finance-mcp + namespace: team1 + labels: + app.kubernetes.io/name: finance-mcp + app.kubernetes.io/part-of: finance-sparc-demo +spec: + selector: + app.kubernetes.io/name: finance-mcp + ports: + - port: 8888 + targetPort: 8888 + protocol: TCP + name: http + type: ClusterIP diff --git a/authbridge/demos/finance-sparc/k8s/ollama.yaml b/authbridge/demos/finance-sparc/k8s/ollama.yaml new file mode 100644 index 000000000..dc1f4fecf --- /dev/null +++ b/authbridge/demos/finance-sparc/k8s/ollama.yaml @@ -0,0 +1,95 @@ +# In-cluster Ollama for the demo — provides the agent's reasoning LLM on the +# watsonx path WITHOUT depending on host networking. The model is staged onto +# the kind node at /opt/ollama-models (by scripts/stage-ollama-model.sh) and +# mounted here, so no large pull is needed. (The PROVIDER=ollama path uses the +# host Ollama instead and doesn't deploy this workload.) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ollama + namespace: team1 + labels: + app.kubernetes.io/name: ollama + app.kubernetes.io/part-of: finance-sparc-demo +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: ollama + template: + metadata: + labels: + app.kubernetes.io/name: ollama + app.kubernetes.io/part-of: finance-sparc-demo + kagenti.io/inject: disabled + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: ollama + image: ollama/ollama:0.30.2 + imagePullPolicy: IfNotPresent + env: + - name: OLLAMA_HOST + value: "0.0.0.0" + - name: OLLAMA_KEEP_ALIVE + value: "24h" + # Run as a non-root uid: keep HOME (keys/history) on a writable + # emptyDir and read the staged models from the mounted hostPath. + - name: HOME + value: /home/ollama + - name: OLLAMA_MODELS + value: /models + ports: + - containerPort: 11434 + name: http + volumeMounts: + - name: models + mountPath: /models + readOnly: true + - name: home + mountPath: /home/ollama + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readinessProbe: + httpGet: + path: /api/tags + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 12 + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + memory: 8Gi + volumes: + - name: models + hostPath: + path: /opt/ollama-models + type: Directory + - name: home + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: ollama + namespace: team1 + labels: + app.kubernetes.io/name: ollama + app.kubernetes.io/part-of: finance-sparc-demo +spec: + selector: + app.kubernetes.io/name: ollama + ports: + - port: 11434 + targetPort: http + name: http diff --git a/authbridge/demos/finance-sparc/k8s/sparc-patch.yaml b/authbridge/demos/finance-sparc/k8s/sparc-patch.yaml new file mode 100644 index 000000000..5a5e5927a --- /dev/null +++ b/authbridge/demos/finance-sparc/k8s/sparc-patch.yaml @@ -0,0 +1,53 @@ +# SPARC plugin-pipeline patch fragment. +# +# NOT a Kubernetes manifest. scripts/patch-sparc-config.sh merges these +# additions into the operator-rendered authbridge ConfigMap +# (authbridge-config-finance-agent in team1): +# - prepend a2a-parser to inbound (so the session carries the user turn) +# - append inference-parser + mcp-parser + sparc to outbound +# inference-parser: captures the agent's LLM call (messages incl. system +# prompt + tool specs) into the session — SPARC's generic +# conversation/tool-spec source +# mcp-parser: parses the outbound MCP tools/call (the tool call) +# sparc: reflects and enforces (mcp mode → MCP tool result) + +inbound_prepend: + - name: a2a-parser + +outbound_append: + - name: inference-parser + - name: mcp-parser + - name: sparc + config: + # The in-cluster SPARC reflection service. The plugin POSTs to {endpoint}/reflect. + # ${...} placeholders are substituted by scripts/patch-sparc-config.sh. + reflector_endpoint: "${SPARC_REFLECTOR_ENDPOINT}" + + # Gate the MCP tools/call and return the clarification as an MCP tool + # result (the kagenti norm). Use "inference" for non-MCP agents. + enforcement: "mcp" + + # fast_track = hallucination + function-selection checks. + track: "fast_track" + + # Return SPARC's reflection to the agent on a reject (reflection mode): + # observe = log only; reflect = return the clarification; deny = hard block. + on_reject_action: "reflect" + + # Escalate a reject to a hard deny when SPARC's score (1=worst..5=best) is + # this low. 0 disables (reflect handles all rejects). + deny_score_threshold: 0 + + # Don't reflect on trivial read-only tools — saves a round-trip. + skip_tools: + - "list_currencies" + + # SPARC is a quality gate, not auth: if the service is unreachable, allow + # and log rather than break the agent. + fail_policy: "open" + + # Max time the plugin waits for a reflection verdict. watsonx answers in + # ~5s; a CPU-only in-cluster Ollama is slower, so the Makefile raises this + # for PROVIDER=ollama. The plugin reflects out-of-band, so this is NOT + # bounded by the forward proxy's upstream timeout. + timeout_ms: ${SPARC_TIMEOUT_MS} diff --git a/authbridge/demos/finance-sparc/k8s/sparc-service.yaml b/authbridge/demos/finance-sparc/k8s/sparc-service.yaml new file mode 100644 index 000000000..a99bb6091 --- /dev/null +++ b/authbridge/demos/finance-sparc/k8s/sparc-service.yaml @@ -0,0 +1,82 @@ +# SPARC reflection service for the demo, deployed into kagenti-system. +# Its config (provider/model/track) is supplied by the `sparc-service-config` +# ConfigMap and the optional `sparc-watsonx` Secret, both created by the +# Makefile (provider-aware: watsonx or ollama). In production this ships via the +# kagenti Helm chart (components.sparcService.enabled=true). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sparc-service + namespace: kagenti-system + labels: + app.kubernetes.io/name: sparc-service + app.kubernetes.io/part-of: finance-sparc-demo +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: sparc-service + template: + metadata: + labels: + app.kubernetes.io/name: sparc-service + app.kubernetes.io/part-of: finance-sparc-demo + kagenti.io/inject: disabled + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: sparc-service + image: sparc-service:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8090 + name: http + envFrom: + - configMapRef: + name: sparc-service-config + - secretRef: + name: sparc-watsonx + optional: true + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readinessProbe: + httpGet: + path: /readyz + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 12 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 5 + periodSeconds: 20 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: sparc-service + namespace: kagenti-system + labels: + app.kubernetes.io/name: sparc-service + app.kubernetes.io/part-of: finance-sparc-demo +spec: + selector: + app.kubernetes.io/name: sparc-service + ports: + - port: 8090 + targetPort: http + name: http diff --git a/authbridge/demos/finance-sparc/scripts/drive-demo.sh b/authbridge/demos/finance-sparc/scripts/drive-demo.sh new file mode 100644 index 000000000..7ae243d8b --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/drive-demo.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Drive the finance-sparc demo end-to-end through REAL inbound auth (no jwt +# bypass). Gets a user token from Keycloak (ROPC), then sends the two-turn +# refund scenario to the agent through its AuthBridge sidecar, and prints the +# SPARC verdicts recorded in the session. +# +# Usage: drive-demo.sh [namespace] [agent] +set -euo pipefail + +NS=${1:-team1} +AGENT=${2:-finance-agent} +REALM=${KEYCLOAK_REALM:-kagenti} +ROPC_CLIENT_ID=${ROPC_CLIENT_ID:-finance-sparc-e2e} +USER=${DEMO_USER:-alice} +PASS=${DEMO_PASS:-alice123} +CTX="finance-demo-$(date +%s)" +PORT=${AGENT_PORT:-18080} +KCPORT=${KC_PORT:-18081} + +say() { printf '\n\033[1m%s\033[0m\n' "$*"; } + +# Reach Keycloak via kubectl port-forward (portable; doesn't depend on the +# cluster ingress / host port-forward). Keycloak's fixed KC_HOSTNAME makes the +# token's issuer correct regardless of how it's reached. +say "[1/4] Obtaining a user token from Keycloak (via port-forward) ..." +kubectl -n keycloak port-forward svc/keycloak-service "$KCPORT:8080" >/tmp/finance-kc-pf.log 2>&1 & +KCPF=$!; trap 'kill $KCPF 2>/dev/null || true' EXIT +sleep 4 +TOKEN=$(curl -s -m 20 "http://localhost:$KCPORT/realms/$REALM/protocol/openid-connect/token" \ + -d grant_type=password -d "client_id=$ROPC_CLIENT_ID" \ + -d "username=$USER" -d "password=$PASS" -d scope=openid \ + | python3 -c 'import json,sys; print(json.load(sys.stdin).get("access_token",""))' 2>/dev/null || true) +if [[ -z "$TOKEN" ]]; then + echo "ERROR: could not get a token. Did you run 'make setup-keycloak'?" >&2 + cat /tmp/finance-kc-pf.log >&2 2>/dev/null || true + exit 1 +fi +echo " got token ($(echo -n "$TOKEN" | wc -c | tr -d ' ') chars)" + +say "[2/4] Port-forwarding the agent ..." +kubectl -n "$NS" port-forward "svc/$AGENT" "$PORT:8080" >/tmp/finance-pf.log 2>&1 & +PF=$!; trap 'kill $PF $KCPF 2>/dev/null || true' EXIT +sleep 4 + +send() { # $1=turn id, $2=text + curl -s -m "${TURN_TIMEOUT:-240}" -X POST "http://localhost:$PORT/" \ + -H 'Content-Type: application/json' -H "Authorization: Bearer $TOKEN" \ + -d "$(python3 -c 'import json,sys; print(json.dumps({"jsonrpc":"2.0","id":sys.argv[1],"method":"message/send","params":{"message":{"role":"user","parts":[{"kind":"text","text":sys.argv[2]}],"contextId":sys.argv[3]}}}))' "$1" "$2" "$CTX")" +} +reply() { python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); r=d.get("result",{}); print(r.get("status",{}).get("message",{}).get("parts",[{}])[0].get("text") or ("ERROR: "+json.dumps(d)))' "$1"; } + +say "[3/4] Turn 1 — descriptive refund, NO transaction id (expect SPARC to ask for the exact id):" +echo ' user> Refund my duplicate $450 subscription charge from last week.' +send 1 'Refund my duplicate $450 subscription charge from last week.' > /tmp/finance-t1.json +echo " agent> $(reply /tmp/finance-t1.json)" + +say "[4/4] Turn 2 — supply the exact id TX4827 (expect SPARC approve -> refund):" +echo ' user> The transaction id is TX4827. Please proceed.' +send 2 'The transaction id is TX4827. Please proceed with the refund, reason: duplicate charge.' > /tmp/finance-t2.json +echo " agent> $(reply /tmp/finance-t2.json)" + +say "SPARC verdicts recorded for this session:" +SID="$CTX" +kubectl -n "$NS" port-forward "deploy/$AGENT" 19094:9094 >/tmp/finance-pf94.log 2>&1 & +PF94=$!; trap 'kill $PF $PF94 $KCPF 2>/dev/null || true' EXIT +sleep 3 +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +curl -s -m 10 "http://localhost:19094/v1/sessions/$SID" 2>/dev/null | python3 "$SCRIPT_DIR/show-verdicts.py" +echo +echo "Done. Inspect the full pipeline with: make show-result" diff --git a/authbridge/demos/finance-sparc/scripts/host-setup.sh b/authbridge/demos/finance-sparc/scripts/host-setup.sh new file mode 100644 index 000000000..908eb5c5f --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/host-setup.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Optional network preflight for the demo — generic, no machine-specific tweaks. +# +# On healthy networks this is a pure no-op: it just checks that the kind node +# can reach the public internet (TLS), which it needs for image pulls and (when +# PROVIDER=watsonx) for the SPARC service to reach watsonx. +# +# A few links — some cellular hotspots and VPNs — silently blackhole PMTUD: they +# drop large TCP segments without sending ICMP "fragmentation needed", so TLS +# handshakes hang. The portable, vendor-neutral remedy is TCP MSS clamping. We +# apply it ONLY if the egress check fails, and ONLY on the kind node itself +# (standard iptables, no host/VM privileges). If your network is healthy you'll +# see "egress healthy" and nothing is changed. +# +# The demo's reasoning + reflection LLMs run in-cluster (see k8s/ollama.yaml), +# so no host-Ollama reachability or host networking is required. +set -euo pipefail + +CLUSTER=${KIND_CLUSTER_NAME:-kagenti} +NODE="${CLUSTER}-control-plane" +PROBE_URL=${EGRESS_PROBE_URL:-https://us-south.ml.cloud.ibm.com} +say() { printf '\033[1m%s\033[0m\n' "$*"; } + +# Probe egress from inside the node (no extra image pulls). Success = we got ANY +# HTTP response (the TLS endpoint is reachable); we do NOT require a 2xx, since a +# bare TLS endpoint often answers 4xx. Only a DNS/connect failure (empty code) +# counts as a blackhole. `curl` without -f returns 0 even on 4xx. +egress_ok() { + local code + code=$(docker exec "$NODE" sh -c \ + "curl -s -m 8 -o /dev/null -w '%{http_code}' '$PROBE_URL' 2>/dev/null" 2>/dev/null || true) + [ -n "$code" ] && [ "$code" != "000" ] +} + +clamp_node() { + for chain in POSTROUTING FORWARD OUTPUT; do + docker exec "$NODE" iptables -t mangle -C "$chain" \ + -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu 2>/dev/null \ + || docker exec "$NODE" iptables -t mangle -A "$chain" \ + -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu 2>/dev/null || true + done +} + +say "[net] checking cluster internet egress (TLS) ..." +if egress_ok; then + echo " egress healthy — no changes made." +else + echo " egress looks blackholed (likely a hotspot/VPN PMTUD issue); applying a portable TCP MSS clamp on the node ..." + clamp_node + if egress_ok; then echo " egress restored via MSS clamp."; else + echo " WARNING: egress still failing after clamp; image pulls / watsonx may be slow or fail on this network." >&2 + fi +fi diff --git a/authbridge/demos/finance-sparc/scripts/launch-abctl.sh b/authbridge/demos/finance-sparc/scripts/launch-abctl.sh new file mode 100644 index 000000000..f0a7ce2c0 --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/launch-abctl.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Launch abctl against the finance-agent's authbridge sidecar session API. +# abctl ships a Namespaces → Pods picker that spawns its own kubectl +# port-forward; this wrapper just verifies prerequisites and runs it. +set -euo pipefail + +NAMESPACE=${1:-team1} +AGENT_NAME=${2:-finance-agent} +ABCTL_BIN=${ABCTL_BIN:-/tmp/abctl-finance-sparc-demo} + +if [[ ! -x "$ABCTL_BIN" ]]; then + echo "ERROR: abctl binary not found at $ABCTL_BIN." >&2 + echo " Run \`make show-result\` (which depends on build-abctl) first." >&2 + exit 1 +fi +if ! kubectl -n "$NAMESPACE" get deploy "$AGENT_NAME" >/dev/null 2>&1; then + echo "ERROR: deployment $NAMESPACE/$AGENT_NAME not found. Run 'make demo' first." >&2 + exit 1 +fi +"$ABCTL_BIN" diff --git a/authbridge/demos/finance-sparc/scripts/patch-sparc-config.sh b/authbridge/demos/finance-sparc/scripts/patch-sparc-config.sh new file mode 100644 index 000000000..a1845f486 --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/patch-sparc-config.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# Patch the operator-rendered authbridge ConfigMap to add the SPARC +# pipeline (a2a-parser inbound + inference-parser + mcp-parser + sparc +# outbound). +# +# Usage: patch-sparc-config.sh +# +# The kagenti operator creates `authbridge-config-` when the agent +# pod is admitted (default pipeline: jwt-validation inbound, token-exchange +# outbound). This script extracts that config, merges k8s/sparc-patch.yaml +# via scripts/pipeline-merge.py, re-applies it, and blocks until the +# running authbridge process reports the matching config SHA (handles both +# the hot-reload and pod-roll convergence paths). +set -euo pipefail + +NAMESPACE=${1:-team1} +AGENT_NAME=${2:-finance-agent} +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +DEMO_DIR=$(dirname "$SCRIPT_DIR") +PATCH_FILE="$DEMO_DIR/k8s/sparc-patch.yaml" +CM_NAME="authbridge-config-$AGENT_NAME" + +# Resolve a python with PyYAML: prefer system python3, else use uv ephemerally. +if python3 -c 'import yaml' 2>/dev/null; then + PY="python3" +elif command -v uv >/dev/null 2>&1; then + PY="uv run --quiet --with pyyaml python3" +else + echo "ERROR: need PyYAML (pip3 install --user pyyaml) or uv on PATH." >&2 + exit 1 +fi + +# Render ${...} placeholders in the patch fragment (provider-tunable knobs). +# Defaults keep watsonx behavior identical to before. +SPARC_TIMEOUT_MS=${SPARC_TIMEOUT_MS:-30000} +SPARC_REFLECTOR_ENDPOINT=${SPARC_REFLECTOR_ENDPOINT:-http://sparc-service.kagenti-system.svc.cluster.local:8090} +RENDERED_PATCH=$(mktemp) +trap 'rm -f "$RENDERED_PATCH"' EXIT +sed -e "s|\${SPARC_TIMEOUT_MS}|${SPARC_TIMEOUT_MS}|g" \ + -e "s|\${SPARC_REFLECTOR_ENDPOINT}|${SPARC_REFLECTOR_ENDPOINT}|g" \ + "$PATCH_FILE" > "$RENDERED_PATCH" +PATCH_FILE="$RENDERED_PATCH" +echo "[*] SPARC reflection: endpoint=$SPARC_REFLECTOR_ENDPOINT timeout_ms=$SPARC_TIMEOUT_MS" + +if ! kubectl -n "$NAMESPACE" get configmap "$CM_NAME" >/dev/null 2>&1; then + echo "ERROR: ConfigMap $NAMESPACE/$CM_NAME not found." >&2 + echo " The operator creates it when the agent pod is admitted." >&2 + echo " Check: kubectl -n $NAMESPACE get pods -l app.kubernetes.io/name=$AGENT_NAME" >&2 + exit 1 +fi + +echo "[*] Merging SPARC additions into $CM_NAME ..." +CURRENT_YAML=$(kubectl -n "$NAMESPACE" get configmap "$CM_NAME" -o jsonpath='{.data.config\.yaml}') +MERGED_YAML=$(printf '%s' "$CURRENT_YAML" | $PY "$SCRIPT_DIR/pipeline-merge.py" "$PATCH_FILE") + +if [[ -z "$MERGED_YAML" ]]; then + echo "ERROR: merge produced empty output" >&2 + exit 1 +fi + +show_plugins() { + $PY -c ' +import yaml, sys +c = yaml.safe_load(sys.stdin) or {} +for d in ("inbound", "outbound"): + names = [p["name"] for p in c.get("pipeline", {}).get(d, {}).get("plugins", [])] + print(f" {d}: {names}") +' +} + +if [[ "$CURRENT_YAML" == "$MERGED_YAML" ]]; then + echo "[*] $CM_NAME already contains the SPARC pipeline — nothing to patch." + printf '%s' "$CURRENT_YAML" | show_plugins + exit 0 +fi + +echo "[*] Applying patched ConfigMap ..." +TMP_CONFIG=$(mktemp) +trap 'rm -f "$TMP_CONFIG" "$RENDERED_PATCH"' EXIT +printf '%s' "$MERGED_YAML" >"$TMP_CONFIG" +kubectl -n "$NAMESPACE" create configmap "$CM_NAME" \ + --from-file=config.yaml="$TMP_CONFIG" --dry-run=client -o yaml | kubectl apply -f - + +echo "[*] Patched. Active plugins now:" +printf '%s' "$MERGED_YAML" | show_plugins + +# Block until the running authbridge process reports the SHA we applied. +WANT_SHA=$(printf '%s' "$MERGED_YAML" | sha256sum | awk '{print $1}') +TIMEOUT=${RELOAD_TIMEOUT:-180} +DEADLINE=$(( $(date +%s) + TIMEOUT )) +echo "[*] Waiting for authbridge to load the patched config (timeout ${TIMEOUT}s)" +echo " target SHA: $WANT_SHA" +ACTIVE_SHA="" +while [[ $(date +%s) -lt $DEADLINE ]]; do + ACTIVE_SHA=$(kubectl -n "$NAMESPACE" exec deploy/"$AGENT_NAME" -c authbridge-proxy -- \ + wget -q -O - http://localhost:9093/reload/status 2>/dev/null | \ + python3 -c 'import json,sys +try: print(json.load(sys.stdin).get("active_config_sha256","")) +except Exception: pass' 2>/dev/null || true) + if [[ "$ACTIVE_SHA" == "$WANT_SHA" ]]; then + echo "[*] Active config SHA matches — SPARC pipeline is live." + exit 0 + fi + sleep 3 +done + +echo "ERROR: authbridge active config did not match patched SHA within ${TIMEOUT}s." >&2 +echo " want: $WANT_SHA" >&2 +echo " last active: ${ACTIVE_SHA:-}" >&2 +kubectl -n "$NAMESPACE" logs deploy/"$AGENT_NAME" -c authbridge-proxy --tail=20 >&2 || true +exit 1 diff --git a/authbridge/demos/finance-sparc/scripts/pipeline-merge.py b/authbridge/demos/finance-sparc/scripts/pipeline-merge.py new file mode 100644 index 000000000..fdf97703b --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/pipeline-merge.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Merge pipeline additions into the operator-rendered authbridge config.yaml. + +Generic over the plugin set: the patch file (argv[1]) declares +`inbound_prepend` / `outbound_append` lists of plugin entries; this merges +them into the operator's config (read from stdin) and writes the result to +stdout. Idempotent — entries matched by `name` are not duplicated. + +(Identical in shape to demos/ibac/scripts/ibac-merge.py; kept demo-local so +the finance-sparc demo is self-contained.) +""" + +import sys + +import yaml + + +def main() -> int: + if len(sys.argv) != 2: + sys.stderr.write("usage: pipeline-merge.py \n") + return 2 + + operator = yaml.safe_load(sys.stdin) or {} + with open(sys.argv[1]) as f: + patch = yaml.safe_load(f) or {} + + pipeline = operator.setdefault("pipeline", {}) + in_plugins = pipeline.setdefault("inbound", {}).setdefault("plugins", []) + out_plugins = pipeline.setdefault("outbound", {}).setdefault("plugins", []) + in_names = {p.get("name") for p in in_plugins} + out_names = {p.get("name") for p in out_plugins} + + # Reverse-then-prepend preserves the patch's natural order at the front. + for entry in reversed(patch.get("inbound_prepend", []) or []): + if entry.get("name") not in in_names: + in_plugins.insert(0, entry) + in_names.add(entry["name"]) + + for entry in patch.get("outbound_append", []) or []: + if entry.get("name") not in out_names: + out_plugins.append(entry) + out_names.add(entry["name"]) + + sys.stdout.write(yaml.safe_dump(operator, default_flow_style=False, sort_keys=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/authbridge/demos/finance-sparc/scripts/setup_keycloak_finance.py b/authbridge/demos/finance-sparc/scripts/setup_keycloak_finance.py new file mode 100644 index 000000000..2aafc1b0b --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/setup_keycloak_finance.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Keycloak setup for the finance-sparc demo. + +Makes the finance-agent's inbound jwt-validation pass for tokens minted by the +demo (scripted ROPC) and by the kagenti UI, by creating an audience client +scope that puts the agent's SPIFFE id in the token `aud`, and a public +direct-access (ROPC) client + a demo user `alice`. + +Idempotent. Run via: uv run --with python-keycloak python setup_keycloak_finance.py +Env overrides: KEYCLOAK_URL, KEYCLOAK_REALM, KEYCLOAK_ADMIN_USERNAME/PASSWORD, +UI_CLIENT_ID, NAMESPACE, AGENT_SA. +""" +import os +import sys + +from keycloak import KeycloakAdmin + +KEYCLOAK_URL = os.environ.get("KEYCLOAK_URL", "http://keycloak.localtest.me:8080") +KEYCLOAK_REALM = os.environ.get("KEYCLOAK_REALM", "kagenti") +ADMIN_USER = os.environ.get("KEYCLOAK_ADMIN_USERNAME", "admin") +ADMIN_PASS = os.environ.get("KEYCLOAK_ADMIN_PASSWORD", "admin") +UI_CLIENT_ID = os.environ.get("UI_CLIENT_ID", "kagenti") +SPIFFE_TRUST_DOMAIN = os.environ.get("SPIFFE_TRUST_DOMAIN", "localtest.me") +NAMESPACE = os.environ.get("NAMESPACE", "team1") +AGENT_SA = os.environ.get("AGENT_SA", "finance-agent") +ROPC_CLIENT_ID = os.environ.get("ROPC_CLIENT_ID", "finance-sparc-e2e") +USER_NAME = "alice" +USER_PASS = "alice123" +USER = {"username": USER_NAME, "email": "alice@example.com", + "enabled": True, "emailVerified": True, "firstName": "Alice", "lastName": "Demo"} + + +def main() -> int: + agent_spiffe = f"spiffe://{SPIFFE_TRUST_DOMAIN}/ns/{NAMESPACE}/sa/{AGENT_SA}" + scope_name = f"agent-{NAMESPACE}-{AGENT_SA}-aud" + print(f"Keycloak: {KEYCLOAK_URL} realm={KEYCLOAK_REALM} agent aud={agent_spiffe}") + + kc = KeycloakAdmin(server_url=KEYCLOAK_URL, username=ADMIN_USER, password=ADMIN_PASS, + realm_name=KEYCLOAK_REALM, user_realm_name="master") + + # 1) audience client scope + mapper → agent SPIFFE id in aud + scope_id = next((s["id"] for s in kc.get_client_scopes() if s["name"] == scope_name), None) + if not scope_id: + scope_id = kc.create_client_scope({ + "name": scope_name, "protocol": "openid-connect", + "attributes": {"include.in.token.scope": "true", "display.on.consent.screen": "false"}, + }, skip_exists=True) + print(f" client scope {scope_name} -> {scope_id}") + try: + kc.add_mapper_to_client_scope(scope_id, { + "name": scope_name + "-aud-mapper", "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "config": {"included.custom.audience": agent_spiffe, + "id.token.claim": "false", "access.token.claim": "true"}, + }) + except Exception as e: + print(f" (mapper exists or: {e})") + try: + kc.add_default_default_client_scope(scope_id) + print(f" '{scope_name}' is now a realm default scope") + except Exception as e: + print(f" (realm default: {e})") + + # 2) public ROPC client for scripted drive + if not kc.get_client_id(ROPC_CLIENT_ID): + kc.create_client({"clientId": ROPC_CLIENT_ID, "name": "finance-sparc E2E (direct access)", + "enabled": True, "publicClient": True, "standardFlowEnabled": True, + "directAccessGrantsEnabled": True}, skip_exists=True) + ropc_internal = kc.get_client_id(ROPC_CLIENT_ID) + for label, cid in [("ROPC " + ROPC_CLIENT_ID, ropc_internal), + ("UI " + UI_CLIENT_ID, kc.get_client_id(UI_CLIENT_ID))]: + if cid: + try: + kc.add_client_default_client_scope(cid, scope_id, {}) + print(f" added aud scope as default on {label}") + except Exception as e: + print(f" ({label} default scope: {e})") + else: + print(f" note: client {label} not found (ok)") + + # 3) demo user alice (admin role → backend operator role for /api/chat) + uid = kc.get_user_id(USER_NAME) + if not uid: + uid = kc.create_user(USER, exist_ok=True) + kc.set_user_password(uid, USER_PASS, temporary=False) + try: + kc.assign_realm_roles(uid, [kc.get_realm_role("admin")]) + except Exception as e: + print(f" (assign admin role: {e})") + print(f" user alice ready ({uid})") + + print("Keycloak setup complete.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/authbridge/demos/finance-sparc/scripts/show-verdicts.py b/authbridge/demos/finance-sparc/scripts/show-verdicts.py new file mode 100644 index 000000000..d089551db --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/show-verdicts.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Print the SPARC plugin verdicts from a session snapshot (read on stdin).""" +import json +import sys + +try: + data = json.load(sys.stdin) +except Exception: + print(" (no session events yet)") + sys.exit(0) + +seen = False +for event in data.get("events", []): + for inv in (event.get("invocations") or {}).get("outbound", []): + if inv.get("plugin") == "sparc" and inv.get("action") in ("allow", "modify", "deny", "observe"): + det = inv.get("details", {}) + seen = True + score = det.get("score") + score_s = f" score={score}" if score not in (None, "") else "" + print( + " SPARC {}/{} tool={}{}".format( + inv.get("action"), inv.get("reason"), det.get("tool"), score_s + ) + ) +if not seen: + print(" (no sparc verdicts found in session; check `make logs-sparc`)") diff --git a/authbridge/demos/finance-sparc/scripts/stage-ollama-model.sh b/authbridge/demos/finance-sparc/scripts/stage-ollama-model.sh new file mode 100644 index 000000000..54bd13fb4 --- /dev/null +++ b/authbridge/demos/finance-sparc/scripts/stage-ollama-model.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Stage an Ollama model onto the kind node for the in-cluster Ollama (mounted at +# /opt/ollama-models via hostPath). Copies the model from the host's local +# Ollama store with `docker cp` (local, no network). If the model isn't present +# on the host, the in-cluster Ollama will pull it on first use instead. +# +# Usage: stage-ollama-model.sh [model] (default llama3.2:3b) +set -euo pipefail + +MODEL=${1:-${OLLAMA_MODEL:-llama3.2:3b}} +CLUSTER=${KIND_CLUSTER_NAME:-kagenti} +NODE="${CLUSTER}-control-plane" +STORE="${OLLAMA_MODELS_DIR:-$HOME/.ollama/models}" +REPO="${MODEL%%:*}"; TAG="${MODEL##*:}" +# Ollama library models live under registry.ollama.ai/library//. +case "$REPO" in */*) NS_PATH="$REPO" ;; *) NS_PATH="library/$REPO" ;; esac +MAN="$STORE/manifests/registry.ollama.ai/$NS_PATH/$TAG" + +if [[ ! -f "$MAN" ]]; then + echo "[stage] model $MODEL not found in host store ($MAN)." + echo "[stage] the in-cluster Ollama will pull it on first use (needs egress)." + exit 0 +fi + +echo "[stage] copying $MODEL from host store to node $NODE:/opt/ollama-models ..." +docker exec "$NODE" mkdir -p "/opt/ollama-models/manifests/registry.ollama.ai/$NS_PATH" /opt/ollama-models/blobs +docker cp "$MAN" "$NODE:/opt/ollama-models/manifests/registry.ollama.ai/$NS_PATH/$TAG" +for d in $(python3 -c "import json;m=json.load(open('$MAN'));print(m['config']['digest']);[print(l['digest']) for l in m['layers']]"); do + blob="sha256-${d#sha256:}" + if ! docker exec "$NODE" test -s "/opt/ollama-models/blobs/$blob" 2>/dev/null; then + docker cp "$STORE/blobs/$blob" "$NODE:/opt/ollama-models/blobs/$blob" + fi +done +echo "[stage] done: $(docker exec "$NODE" du -sh /opt/ollama-models | awk '{print $1}') staged for $MODEL" diff --git a/authbridge/docs/sparc-plugin.md b/authbridge/docs/sparc-plugin.md new file mode 100644 index 000000000..8c4909089 --- /dev/null +++ b/authbridge/docs/sparc-plugin.md @@ -0,0 +1,165 @@ +# SPARC (Pre-Tool Reflection) Plugin + +The `sparc` plugin verifies that an agent's proposed tool call is *grounded* in the +conversation and the available tool specifications before it executes — catching +**hallucinated / ungrounded arguments** (e.g. an invented transaction id) and inappropriate +tool selection. On a reflected reject it returns SPARC's clarification to the agent so the +agent re-asks the user; the bad call never runs. + +SPARC is the `SPARCReflectionComponent` from the +[`agent-lifecycle-toolkit`](https://pypi.org/project/agent-lifecycle-toolkit/) (ALTK) Python +package, served over HTTP by the companion [SPARC reflection service](../sparc-service/README.md). +Because AuthBridge plugins are Go, this plugin calls the service over HTTP (the same shape +`ibac` uses for its judge). All enforcement policy lives in the plugin; the service only +returns SPARC's verdict. + +It is **complementary to [`ibac`](ibac-plugin.md)**: SPARC verifies argument *grounding*; +IBAC verifies *intent alignment* (prompt-injection / exfiltration). + +## Generic by design — works for any agent + +SPARC's three inputs are collected from exactly what the agent produces, with no per-agent +wiring or out-of-band data: + +- **conversation history, including the system prompt** — from the agent's LLM call captured + by `inference-parser` (`pctx.Extensions.Inference.Messages`, every role preserved); +- **tool specifications in OpenAI function-calling format** — from the same call + (`pctx.Extensions.Inference.Tools`); +- **the proposed tool call** — from the outbound MCP `tools/call` (`mcp` mode) or from the LLM + response (`inference` mode). + +If the conversation/tool context isn't available (no `inference-parser`, or session tracking +off), the plugin **skips** (`no_inference_context`) rather than reflect on partial data. + +## Enforcement is format-aware + +The verdict is returned to the agent in the shape it expects, selected by `enforcement`: + +- **`mcp` (default — the kagenti norm).** Gate the outbound MCP `tools/call`. The tool call + comes from the MCP request; conversation + tool specs are correlated from the session's most + recent inference request (robust to LLM streaming). On a reflected reject, return SPARC's + clarification as a **JSON-RPC MCP tool result** (`Reject` + `Violation{Status:200, Body}`); + the agent's MCP client consumes it like any tool output. +- **`inference` (for non-MCP agents).** Gate the agent's LLM response, where all three inputs + are co-located. On a reflected reject, rewrite the completion (via `SetResponseBody`) so the + assistant turn carries the clarification and the tool_call is dropped — norms-correct for any + OpenAI-style agent. + +## On-reject policy + +When SPARC returns `reject`, `on_reject_action` decides, with optional score escalation: + +| `on_reject_action` | Behavior | Session record | +|---|---|---| +| `observe` | Log only, let the call through (shadow). | `observe` / `reject_observed` | +| `reflect` (default) | Return SPARC's clarification to the agent (MCP result or rewritten completion). | `modify` / `reflected` | +| `deny` | Hard block (403 in `mcp` mode; refusal completion in `inference` mode). | `deny` / `blocked` | + +When `deny_score_threshold > 0` and SPARC's `overall_avg_score` (normalized 0..1, higher = better +grounded) is `<=` it, any reject is escalated to `deny`. + +> **`fail_policy` defaults to `open`** — when SPARC is unreachable, times out, or returns +> `decision=error`, the proposed tool call is allowed through (and recorded). This is deliberate +> and **diverges from `ibac`, which fails closed**: SPARC is a grounding *quality gate*, not an +> auth control, so reflector downtime shouldn't take the agent's tools offline. Set +> `fail_policy: closed` when you want tool execution gated on reflector availability (e.g. a +> high-assurance deployment where an unverified call is worse than a failed one). The same policy +> governs the case where the conversation/tool context is missing (no `inference-parser` event): +> `open` skips and records `no_inference_context`; `closed` blocks. + +## Configuration + +```yaml +pipeline: + outbound: + plugins: + - name: sparc + config: + reflector_endpoint: "http://sparc-service.kagenti-system.svc:8090" + enforcement: "mcp" # mcp | inference + track: "fast_track" # fast_track|slow_track|syntax|spec_free|transformations_only + on_reject_action: "reflect" # observe | reflect | deny + deny_score_threshold: 0 # 0 disables; e.g. 2.0 → deny rejects scoring <= 2 + fail_policy: "open" # open=allow+log on SPARC error; closed=block + timeout_ms: 30000 + skip_tools: ["list_*"] # tool-name globs NOT reflected on (e.g. read-only tools) + reflect_tools: [] # if non-empty, ONLY these globs are reflected +``` + +| Field | Required | Default | Description | +|---|---|---|---| +| `reflector_endpoint` | Yes | — | Base URL of the SPARC service; the plugin POSTs to `{endpoint}/reflect`. | +| `reflector_bearer` | No | `""` | Bearer for the service. Empty for in-cluster unauthenticated calls. | +| `enforcement` | No | `mcp` | `mcp` (gate MCP tools/call, return MCP result) or `inference` (gate LLM response, rewrite completion). | +| `track` | No | `fast_track` | SPARC track. `fast_track` = hallucination + function-selection checks. | +| `on_reject_action` | No | `reflect` | `observe` \| `reflect` \| `deny` (see table above). | +| `deny_score_threshold` | No | `0` | Escalate a reject to `deny` when score `<=` this (`[0,1]`; `0` disables). | +| `fail_policy` | No | `open` | On SPARC unreachable / `decision=error`: `open` allows+logs; `closed` blocks. | +| `timeout_ms` | No | `30000` | Per-call reflection timeout. Rejected below `100`. Reflection is **out-of-band** (the plugin calls the SPARC service directly), so this is independent of the forward proxy's upstream timeout — raise it for slow/CPU-bound providers (e.g. an in-cluster Ollama). | +| `skip_tools` | No | — | Tool-name globs (`path.Match`) to NOT reflect on (e.g. trivial read tools). | +| `reflect_tools` | No | — | If non-empty, ONLY reflect tools matching these globs; all others skipped. | +| `bypass_hosts` / `bypass_paths` | No | built-in | Host / path globs skipped without reflecting. | + +### Pipeline composition + +`sparc` declares `RequiresAny: ["inference-parser", "mcp-parser"]`. For the richest, generic +reflection wire all three parsers: + +```yaml +pipeline: + inbound: + plugins: + - name: a2a-parser # seeds the session with the user turn + - name: jwt-validation + outbound: + plugins: + - name: token-exchange + - name: inference-parser # captures messages (incl. system) + tool specs + - name: mcp-parser # parses the tool call (mcp mode) + - name: sparc +``` + +## Observability + +`sparc` records a flat `Invocation` per call (action `allow`/`modify`/`deny`/`observe`, +`reason`, `tool`, `score`) **and** publishes the full structured verdict via the plugin-event +escape-hatch (`Extensions.Custom["sparc/event"]` → `SessionEvent.Plugins["sparc"]`), so abctl +and other consumers can render the decision, score, track, enforcement mode, and per-issue +explanations/corrections. + +## Status codes & reasons + +| Reason | HTTP / effect | Meaning | +|---|---|---| +| `sparc.reflected` | 200 MCP result / rewritten completion | A reject was reflected back to the agent as a clarification. | +| `sparc.blocked` | 403 (mcp) / refusal completion (inference) | A reject handled by `deny` (or escalated via score). | +| `sparc.reflector_unavailable` | 503 / refusal (closed); pass-through (open) | SPARC unreachable or `decision=error`, per `fail_policy`. | +| `sparc.no_inference_context` | 503 (closed); pass-through (open) | No conversation/tool context to ground against, per `fail_policy`. | + +## Security / network posture + +The reflection service is an **in-cluster backend** — same trust model as the authbridge session +API (`:9094`): no ingress, never exposed publicly. Its `/reflect` endpoint is unauthenticated by +default and runs outside the agent-side ambient mesh (`kagenti.io/inject: disabled`), so it makes +its own egress to the LLM provider. Treat the cluster network boundary as the control: restrict +who can reach it with a `NetworkPolicy` (allow only the authbridge sidecars), since any pod that +can POST to `/reflect` can trigger LLM calls billed to the configured credentials. For an extra +hop, set `reflector_bearer` on the plugin and terminate it at the service (or a sidecar). The +service never echoes provider exception text back to callers; details stay in its logs. + +## Limitations & non-goals +- Not an intent/auth control — use `ibac` + `token-exchange` routes / NetworkPolicy for that. +- No principal/identity input — SPARC reflects on `(messages, tool_specs, tool_calls)` only. +- `inference` mode can't see streamed tool_calls (use `mcp` mode for streaming MCP agents). +- Cost: one reflection round-trip per (non-skipped) tool call. + +## See also +- [`sparc-service/README.md`](../sparc-service/README.md) — the reflection service. +- [`demos/finance-sparc/`](../demos/finance-sparc/README.md) — the end-to-end demo. +- [`ibac-plugin.md`](ibac-plugin.md) — the complementary intent control. + +## Files +- `authlib/plugins/sparc/plugin.go` — config, gates, policy, structured event. +- `authlib/plugins/sparc/collect.go` — generic input collection. +- `authlib/plugins/sparc/respond.go` — MCP result + completion-rewrite responders. +- `authlib/plugins/sparc/reflector.go` — HTTP client to the service. diff --git a/authbridge/sparc-service/.dockerignore b/authbridge/sparc-service/.dockerignore new file mode 100644 index 000000000..b1bce9d9d --- /dev/null +++ b/authbridge/sparc-service/.dockerignore @@ -0,0 +1,7 @@ +.venv +__pycache__ +*.pyc +tests +.pytest_cache +*.egg-info +.env diff --git a/authbridge/sparc-service/Dockerfile b/authbridge/sparc-service/Dockerfile new file mode 100644 index 000000000..fa504deb7 --- /dev/null +++ b/authbridge/sparc-service/Dockerfile @@ -0,0 +1,24 @@ +# Kagenti SPARC reflection service. +# Depends on the agent-lifecycle-toolkit (ALTK) package straight from PyPI — no vendoring. +FROM python:3.12-slim AS base + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + LITELLM_LOCAL_MODEL_COST_MAP=True + +WORKDIR /app + +# Install dependencies first for better layer caching. +COPY pyproject.toml README.md ./ +COPY sparc_service ./sparc_service +RUN pip install --upgrade pip && pip install . + +# Drop privileges. +RUN useradd --create-home --uid 10001 sparc +USER sparc + +EXPOSE 8090 + +# PORT/HOST and SPARC_* / WX_* are provided via the Deployment env (ConfigMap/Secret). +CMD ["python", "-m", "sparc_service"] diff --git a/authbridge/sparc-service/README.md b/authbridge/sparc-service/README.md new file mode 100644 index 000000000..93b98c2e3 --- /dev/null +++ b/authbridge/sparc-service/README.md @@ -0,0 +1,136 @@ +# Kagenti SPARC reflection service + +A thin, in-process HTTP wrapper around the **SPARC** pre-tool reflection component +(`altk.pre_tool.sparc.SPARCReflectionComponent`) from the +[`agent-lifecycle-toolkit`](https://pypi.org/project/agent-lifecycle-toolkit/) (ALTK) +PyPI package. + +AuthBridge's Go [`sparc` plugin](../authlib/plugins/sparc) calls this service to decide +whether a proposed tool call is **grounded** in the conversation and the available tool +specs. SPARC catches hallucinated / ungrounded arguments (e.g. an invented transaction +ID) and inappropriate function selection. + +This service is deliberately *just* a reflection wrapper: + +- It consumes ALTK **directly from PyPI** — no vendored copy of the SPARC source. +- It calls SPARC **in process** — no relay queue, observer, or host worker (those exist + only in the original `ibac` reference repo as a `kind`-egress workaround). +- It returns SPARC's verdict **faithfully**. All enforcement policy (observe / inject / + deny, score thresholds) lives in the AuthBridge plugin, not here. + +## API + +### `POST /reflect` + +Request: + +```jsonc +{ + "messages": [ { "role": "user", "content": "Refund transaction TX482." } ], + "tool_specs": [ { "type": "function", "function": { "name": "get_transaction", ... } } ], + "tool_calls": [ { "id": "c1", "type": "function", + "function": { "name": "get_transaction", + "arguments": "{\"transaction_id\": \"TX4821\"}" } } ], + "session_id": "optional-correlation-id", + "track": "fast_track" // optional per-request override +} +``` + +Response: + +```jsonc +{ + "decision": "approve | reject | error", + "issues": [ { "issue_type": "...", "metric_name": "...", + "explanation": "...", "correction": { ... } } ], + "overall_avg_score": 0.4, // SPARC grounding score, normalized 0..1 (higher = better), when available + "execution_time_ms": 4600.0, + "raw_pipeline_result": { ... } // present only if SPARC_INCLUDE_RAW_RESPONSE=true +} +``` + +### `GET /healthz` — liveness. `GET /readyz` — readiness (config valid + component buildable). + +On failure `/reflect` returns a stable message (`{"error": "reflection failed"}`, or `400` for a +bad request such as an unsupported `track`); the underlying provider exception is logged, never +returned to the caller — provider errors can embed endpoints or credentials. + +## Security / network posture + +Deploy this as an **in-cluster backend only** — no ingress, never public. `/reflect` is +unauthenticated by default, so the network boundary is the control: restrict callers with a +`NetworkPolicy` (allow only the authbridge sidecars). Any pod that can POST to `/reflect` can +trigger LLM calls billed to the configured credentials. For an additional hop, set the plugin's +`reflector_bearer` and verify it here (or at a fronting sidecar). The service runs as a non-root +user and makes its own egress to the LLM provider; it stays out of the agent-side ambient mesh. + +## Configuration (environment) + +**watsonx is the default.** Switching providers is a config change — no rebuild. `watsonx` +and `ollama` use ALTK's provider-native clients; `openai`, `azure` (Azure OpenAI), and the +generic `litellm` provider use ALTK's generic LiteLLM client, so **any provider LiteLLM/ALTK +supports** works by setting the model string and (optionally) extra client kwargs. + +| Variable | Default | Notes | +| --- | --- | --- | +| `SPARC_LLM_PROVIDER` | `watsonx` | `watsonx` \| `ollama` \| `openai` \| `azure` \| `litellm` (generic) | +| `SPARC_MODEL` | per-provider | watsonx: `mistral-large-2512`, ollama: `llama3.2:3b`, openai: `gpt-4o-mini`. **Required** for `azure`/`litellm` (e.g. `azure/`, `anthropic/claude-3-5-sonnet`, `gemini/gemini-1.5-pro`, `bedrock/...`). | +| `SPARC_LLM_KWARGS_JSON` | — | JSON object of extra LiteLLM client kwargs for `openai`/`azure`/`litellm`, e.g. `{"api_base":"...","api_version":"...","api_key":"..."}`. | +| `SPARC_LLM_REGISTRY_ID` | — | Advanced: override the ALTK client registry id directly (any ALTK client). | +| `SPARC_TRACK` | `fast_track` | `fast_track` \| `slow_track` \| `syntax` \| `spec_free` \| `transformations_only` | +| `SPARC_LLM_TIMEOUT` | `120` | seconds | +| `SPARC_RETRIES` | `3` | SPARC retries | +| `SPARC_MAX_PARALLEL` | `2` | SPARC metric parallelism | +| `SPARC_INCLUDE_RAW_RESPONSE` | `true` | include `raw_pipeline_result` in responses | +| `WX_API_KEY` / `WX_PROJECT_ID` / `WX_URL` | — | watsonx creds (also accepts `WATSONX_*`) | +| `OLLAMA_BASE_URL` | `http://localhost:11434` | ollama endpoint | +| `OPENAI_API_KEY` / `OPENAI_BASE_URL` | — | openai creds (LiteLLM also reads provider keys from env: `OPENAI_API_KEY`, `AZURE_API_KEY`, `ANTHROPIC_API_KEY`, …) | +| `HOST` / `PORT` | `0.0.0.0` / `8090` | bind address | + +### Provider examples + +```bash +# watsonx (default) +SPARC_LLM_PROVIDER=watsonx WX_API_KEY=... WX_PROJECT_ID=... + +# OpenAI +SPARC_LLM_PROVIDER=openai SPARC_MODEL=gpt-4o-mini OPENAI_API_KEY=sk-... + +# Azure OpenAI +SPARC_LLM_PROVIDER=azure SPARC_MODEL=azure/ \ + SPARC_LLM_KWARGS_JSON='{"api_base":"https://.openai.azure.com","api_version":"2024-06-01","api_key":"..."}' + +# Anything else LiteLLM supports (Anthropic, Gemini, Bedrock, ...) +SPARC_LLM_PROVIDER=litellm SPARC_MODEL=anthropic/claude-3-5-sonnet \ + SPARC_LLM_KWARGS_JSON='{"api_key":"sk-ant-..."}' + +# Local Ollama +SPARC_LLM_PROVIDER=ollama SPARC_MODEL=llama3.2:3b OLLAMA_BASE_URL=http://host:11434 +``` + +## Develop & test + +```bash +python -m venv .venv && . .venv/bin/activate +pip install -e ".[dev]" + +# Fast, offline unit tests (fake SPARC component — no network, no creds): +pytest + +# Opt-in integration test against real watsonx: +RUN_WATSONX_TESTS=1 WX_API_KEY=... WX_PROJECT_ID=... pytest tests/test_watsonx_integration.py -v +``` + +## Run + +```bash +SPARC_LLM_PROVIDER=watsonx WX_API_KEY=... WX_PROJECT_ID=... python -m sparc_service +# or against local Ollama: +SPARC_LLM_PROVIDER=ollama SPARC_MODEL=llama3.2:3b python -m sparc_service +``` + +## Build the image + +```bash +docker build -t kagenti-sparc-service:latest . +``` diff --git a/authbridge/sparc-service/pyproject.toml b/authbridge/sparc-service/pyproject.toml new file mode 100644 index 000000000..fdae7b124 --- /dev/null +++ b/authbridge/sparc-service/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "kagenti-sparc-service" +version = "0.1.0" +description = "Kagenti SPARC reflection service — a thin in-process HTTP wrapper around the SPARC pre-tool reflection component from the agent-lifecycle-toolkit (ALTK)." +readme = "README.md" +requires-python = ">=3.10" +license = { text = "Apache-2.0" } +dependencies = [ + # The SPARC engine itself, consumed directly from PyPI (no vendoring). + "agent-lifecycle-toolkit==0.10.1", + "fastapi>=0.110", + "uvicorn[standard]>=0.27", + "pydantic>=2", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8", + "httpx>=0.27", +] + +[project.scripts] +kagenti-sparc-service = "sparc_service.__main__:main" + +[tool.hatch.build.targets.wheel] +packages = ["sparc_service"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" diff --git a/authbridge/sparc-service/sparc_service/__init__.py b/authbridge/sparc-service/sparc_service/__init__.py new file mode 100644 index 000000000..afbf6b7b3 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/__init__.py @@ -0,0 +1,16 @@ +"""Kagenti SPARC reflection service. + +A thin, in-process HTTP wrapper around the SPARC pre-tool reflection component +(`altk.pre_tool.sparc.SPARCReflectionComponent`) shipped in the +``agent-lifecycle-toolkit`` PyPI package. + +The service exposes a single reflection endpoint that AuthBridge's Go ``sparc`` +plugin calls to decide whether a proposed tool call is grounded in the +conversation and tool specifications. All enforcement policy (observe / inject / +deny, score thresholds) lives in the plugin; this service only returns SPARC's +verdict faithfully. +""" + +__all__ = ["__version__"] + +__version__ = "0.1.0" diff --git a/authbridge/sparc-service/sparc_service/__main__.py b/authbridge/sparc-service/sparc_service/__main__.py new file mode 100644 index 000000000..a0f387a4d --- /dev/null +++ b/authbridge/sparc-service/sparc_service/__main__.py @@ -0,0 +1,16 @@ +"""Console/`python -m sparc_service` entry point.""" + +from __future__ import annotations + +import uvicorn + +from .settings import Settings + + +def main() -> None: + settings = Settings.from_env() + uvicorn.run("sparc_service.api:app", host=settings.host, port=settings.port, log_level="info") + + +if __name__ == "__main__": + main() diff --git a/authbridge/sparc-service/sparc_service/api.py b/authbridge/sparc-service/sparc_service/api.py new file mode 100644 index 000000000..3a7552083 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/api.py @@ -0,0 +1,70 @@ +"""FastAPI application for the SPARC reflection service. + +Endpoints: + POST /reflect — run SPARC on a proposed tool call, return the verdict. + GET /healthz — liveness (always ok if the process is up). + GET /readyz — readiness (config valid and component buildable). +""" + +from __future__ import annotations + +import logging + +from fastapi import FastAPI, HTTPException +from fastapi.concurrency import run_in_threadpool + +from .engine import ReflectionEngine +from .models import ReflectRequest, ReflectResponse +from .settings import Settings + +log = logging.getLogger(__name__) + + +def create_app(engine: ReflectionEngine | None = None) -> FastAPI: + """Build the FastAPI app. Inject ``engine`` in tests; defaults to env config.""" + settings = engine.settings if engine is not None else Settings.from_env() + engine = engine or ReflectionEngine(settings) + + app = FastAPI( + title="Kagenti SPARC reflection service", + version="0.1.0", + summary="In-process SPARC pre-tool reflection over HTTP.", + ) + app.state.engine = engine + app.state.settings = settings + + @app.get("/healthz") + def healthz() -> dict[str, object]: + return { + "status": "ok", + "provider": settings.provider, + "model": settings.model, + "track": settings.track, + } + + @app.get("/readyz") + def readyz() -> dict[str, object]: + ok, detail = engine.ready() + if not ok: + raise HTTPException(status_code=503, detail={"status": "not_ready", "reason": detail}) + return {"status": "ready", "provider": settings.provider, "model": settings.model} + + @app.post("/reflect", response_model=ReflectResponse) + async def reflect(request: ReflectRequest) -> ReflectResponse: + # SPARCReflectionComponent.process is synchronous (and CPU/IO bound on the + # LLM call); run it off the event loop so the service stays responsive. + try: + return await run_in_threadpool(engine.reflect, request) + except ValueError as exc: # bad input (e.g. unsupported track) → 400 + raise HTTPException(status_code=400, detail={"error": str(exc)}) from exc + except Exception: + # Reflection failure → 502 so the plugin can apply its fail policy. + # Keep the full error in the service logs; never echo provider + # exception text to the caller — it can embed endpoints/credentials. + log.exception("reflection failed") + raise HTTPException(status_code=502, detail={"error": "reflection failed"}) from None + + return app + + +app = create_app() diff --git a/authbridge/sparc-service/sparc_service/engine.py b/authbridge/sparc-service/sparc_service/engine.py new file mode 100644 index 000000000..053fbf929 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/engine.py @@ -0,0 +1,136 @@ +"""The reflection engine: lazily builds SPARC components and maps their output. + +Components are built lazily and cached *per track* so the service starts even +when the LLM backend is briefly unavailable, switches tracks per request without +rebuilding the common case, and lets unit tests inject a fake component factory +(no network, no credentials). +""" + +from __future__ import annotations + +import logging +import threading +from dataclasses import replace +from typing import Any, Callable + +from .models import ReflectionIssue, ReflectRequest, ReflectResponse +from .providers import build_component +from .settings import SUPPORTED_TRACKS, Settings + +log = logging.getLogger(__name__) + +# A factory takes a track name and returns an object exposing +# ``process(run_input, phase) -> output`` where output has +# ``reflection_result``, ``execution_time_ms`` and ``raw_pipeline_result``. +# The real one is altk's SPARCReflectionComponent. +ComponentFactory = Callable[[str], Any] + + +def _decision_str(decision: Any) -> str: + """Normalize SPARC's decision enum/str to a plain string.""" + return str(getattr(decision, "value", decision)) + + +def _extract_overall_score(raw_pipeline: dict[str, Any] | None) -> float | None: + if not raw_pipeline: + return None + value = raw_pipeline.get("overall_avg_score") + return float(value) if isinstance(value, (int, float)) else None + + +class ReflectionEngine: + """Builds (per track) and invokes SPARC components, mapping IO to wire models.""" + + def __init__( + self, + settings: Settings, + component_factory: ComponentFactory | None = None, + ) -> None: + self._settings = settings + self._factory: ComponentFactory = component_factory or ( + lambda track: build_component(replace(settings, track=track)) + ) + self._lock = threading.Lock() + # Components keyed by track, so a per-request track override doesn't + # rebuild the common case. Most deployments use exactly one track. + self._components: dict[str, Any] = {} + + @property + def settings(self) -> Settings: + return self._settings + + def _component_or_build(self, track: str) -> Any: + with self._lock: + existing = self._components.get(track) + if existing is not None: + return existing + component = self._factory(track) # may raise → surfaced as 503 / not-ready + self._components[track] = component + return component + + def ready(self) -> tuple[bool, str | None]: + """Best-effort readiness: try to build the default-track component once. + + Our own validation errors (missing creds, etc.) are safe to surface + verbatim. A build exception may carry provider text (endpoints, request + details), so it's logged in full and reported as a generic reason. + """ + if self._settings.errors: + return False, "; ".join(self._settings.errors) + try: + self._component_or_build(self._settings.track) + return True, None + except Exception: + log.exception("SPARC component build failed") + return False, "component initialization failed; see service logs" + + def reflect(self, request: ReflectRequest) -> ReflectResponse: + """Run SPARC reflection on a proposed tool call and project the verdict.""" + from altk.core.toolkit import AgentPhase + from altk.pre_tool.core import SPARCReflectionRunInput + + track = request.track or self._settings.track + if track not in SUPPORTED_TRACKS: + raise ValueError(f"unsupported track {track!r}; expected one of {sorted(SUPPORTED_TRACKS)}") + + component = self._component_or_build(track) + + run_input = SPARCReflectionRunInput( + messages=request.messages, + tool_specs=request.tool_specs, + tool_calls=request.tool_calls, + ) + result = component.process(run_input, phase=AgentPhase.RUNTIME) + output = result.output + reflection = output.reflection_result + raw_pipeline = getattr(output, "raw_pipeline_result", None) or {} + + issues = [ + ReflectionIssue( + issue_type=getattr(i, "issue_type", None), + metric_name=getattr(i, "metric_name", None), + explanation=getattr(i, "explanation", None), + correction=getattr(i, "correction", None), + ) + for i in reflection.issues + ] + + decision = _decision_str(reflection.decision) + score = _extract_overall_score(raw_pipeline) + execution_ms = getattr(output, "execution_time_ms", None) + log.info( + "reflect session=%s track=%s decision=%s score=%s ms=%s", + request.session_id or "-", + track, + decision, + f"{score:.2f}" if score is not None else "-", + f"{execution_ms:.1f}" if isinstance(execution_ms, (int, float)) else "-", + ) + + return ReflectResponse( + decision=decision, + issues=issues, + overall_avg_score=score, + execution_time_ms=execution_ms, + raw_pipeline_result=raw_pipeline if self._settings.include_raw_response else None, + ) diff --git a/authbridge/sparc-service/sparc_service/models.py b/authbridge/sparc-service/sparc_service/models.py new file mode 100644 index 000000000..60ed20678 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/models.py @@ -0,0 +1,60 @@ +"""Wire models for the SPARC reflection service. + +The request mirrors SPARC's native input — an OpenAI-style conversation, the +available tool inventory, and the proposed tool call(s). The response is a thin, +faithful projection of SPARC's verdict; enforcement policy is the caller's job. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + + +class ReflectRequest(BaseModel): + """Inputs SPARC needs to judge a proposed tool call.""" + + messages: list[dict[str, Any]] = Field( + default_factory=list, + description="OpenAI-style conversation context (role/content turns).", + ) + tool_specs: list[dict[str, Any]] = Field( + default_factory=list, + description="Available tools in OpenAI function-calling format.", + ) + tool_calls: list[dict[str, Any]] = Field( + ..., + min_length=1, + description="Proposed tool call(s); SPARC evaluates the first.", + ) + session_id: str | None = Field(default=None, description="Opaque correlation id for logs/observability.") + track: str | None = Field( + default=None, + description="Per-request SPARC track override; falls back to the service's SPARC_TRACK.", + ) + + +class ReflectionIssue(BaseModel): + """A single grounding/appropriateness problem SPARC found.""" + + issue_type: str | None = None + metric_name: str | None = None + explanation: str | None = None + correction: Any | None = None + + +class ReflectResponse(BaseModel): + """SPARC's verdict, faithfully projected for the AuthBridge plugin.""" + + decision: str = Field(description="approve | reject | error") + issues: list[ReflectionIssue] = Field(default_factory=list) + overall_avg_score: float | None = Field( + default=None, + description="SPARC's grounding score, normalized 0..1 (higher = better grounded), when available.", + ) + execution_time_ms: float | None = None + raw_pipeline_result: dict[str, Any] | None = Field( + default=None, + description="Full SPARC pipeline detail; included only when SPARC_INCLUDE_RAW_RESPONSE is set.", + ) diff --git a/authbridge/sparc-service/sparc_service/providers.py b/authbridge/sparc-service/sparc_service/providers.py new file mode 100644 index 000000000..6489926b1 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/providers.py @@ -0,0 +1,116 @@ +"""ALTK LLM-client and SPARC-component construction. + +Isolates every import of the heavy ``altk`` package so the rest of the service +(settings, models, API wiring) stays import-light and unit-testable without +network access or LLM credentials. +""" + +from __future__ import annotations + +import os +from typing import Any + +from .settings import Settings + +# Map the configurable track names to altk Track enum members. Resolved lazily. +_TRACK_NAMES = { + "fast_track": "FAST_TRACK", + "slow_track": "SLOW_TRACK", + "syntax": "SYNTAX", + "spec_free": "SPEC_FREE", + "transformations_only": "TRANSFORMATIONS_ONLY", +} + + +def resolve_track(name: str): + """Return the altk ``Track`` enum member for a configured track name.""" + from altk.pre_tool.core import Track + + return getattr(Track, _TRACK_NAMES[name]) + + +def build_llm_client(settings: Settings): + """Construct a validating ALTK LLM client for the configured provider. + + watsonx (default) and ollama use ALTK's provider-native LiteLLM validating + clients. Every other provider — openai, azure (Azure OpenAI), and the + generic ``litellm`` escape hatch — uses ALTK's generic LiteLLM validating + client (``litellm.output_val``), where the model string selects the provider + and any extra client kwargs come from ``SPARC_LLM_KWARGS_JSON``. SPARC + requires a *validating* client, so every branch returns one. + """ + from altk.core.llm import get_llm + + client_cls = get_llm(settings.registry_id) + + # An explicit SPARC_LLM_REGISTRY_ID override means the caller picked the ALTK + # client class directly, so its constructor may not accept the provider-native + # kwargs below (e.g. watsonx's project_id). In that case skip the native + # branches and use the generic LiteLLM-kwargs path, where everything the + # client needs comes from SPARC_LLM_KWARGS_JSON. + native = not settings.llm_registry_id + + if native and settings.provider == "watsonx": + return client_cls( + model_name=settings.model, + api_key=settings.wx_api_key, + project_id=settings.wx_project_id, + api_base=settings.wx_url, + timeout=settings.llm_timeout_seconds, + ) + + if native and settings.provider == "ollama": + # Point every LiteLLM Ollama call at the configured server. We pass + # api_url EXPLICITLY (the ALTK ollama client's param) so ALTK's internal + # metric sub-clients inherit it — relying on the OLLAMA_API_BASE env var + # alone is not enough (those sub-calls otherwise fall back to + # localhost:11434). The env vars are set too as a belt-and-suspenders. + os.environ["OLLAMA_API_BASE"] = settings.ollama_base_url + os.environ["OLLAMA_BASE_URL"] = settings.ollama_base_url + return client_cls( + model_name=settings.model, + api_key=settings.ollama_api_key, + api_url=settings.ollama_base_url, + ) + + # Generic path (openai / azure / litellm, or any provider when + # SPARC_LLM_REGISTRY_ID is set). LiteLLM routes by the model string and reads + # provider API keys from the environment (OPENAI_API_KEY, AZURE_API_KEY, + # ANTHROPIC_API_KEY, ...) when not supplied; SPARC_LLM_KWARGS_JSON supplies + # anything else (api_base, api_version, api_key, deployment, ...). When a + # registry override selects a non-LiteLLM client, supply its constructor + # kwargs via SPARC_LLM_KWARGS_JSON. + lite_kwargs: dict[str, Any] = dict(settings.llm_kwargs) + lite_kwargs.setdefault("timeout", settings.llm_timeout_seconds) + if settings.provider == "openai" and settings.openai_base_url and "api_base" not in lite_kwargs: + lite_kwargs["api_base"] = settings.openai_base_url + return client_cls(model_name=settings.model, **lite_kwargs) + + +def build_component(settings: Settings): + """Construct the SPARC reflection component from settings. + + This performs provider authentication eagerly for watsonx/openai (the + validating clients authenticate on construction), so callers should treat a + raised exception as "not ready". + """ + # Mirror the cost-map flag the reference reflector sets for LiteLLM. + os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + from altk.core.toolkit import ComponentConfig + from altk.pre_tool.core import SPARCExecutionMode + from altk.pre_tool.sparc import SPARCReflectionComponent + + config = ComponentConfig(llm_client=build_llm_client(settings)) + component = SPARCReflectionComponent( + config=config, + track=resolve_track(settings.track), + execution_mode=SPARCExecutionMode.ASYNC, + include_raw_response=settings.include_raw_response, + retries=settings.retries, + max_parallel=settings.max_parallel, + ) + init_error = getattr(component, "_initialization_error", None) + if init_error: + raise RuntimeError(f"SPARC component failed to initialize: {init_error}") + return component diff --git a/authbridge/sparc-service/sparc_service/settings.py b/authbridge/sparc-service/sparc_service/settings.py new file mode 100644 index 000000000..12023b1c4 --- /dev/null +++ b/authbridge/sparc-service/sparc_service/settings.py @@ -0,0 +1,175 @@ +"""Environment-driven configuration for the SPARC reflection service. + +Configuration is intentionally read from the environment (12-factor) so the +service can be tuned via a Kubernetes ConfigMap/Secret without rebuilding the +image. Watsonx credentials follow the same ``WX_*`` (with ``WATSONX_*`` +fallbacks) convention used elsewhere in Kagenti. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field + +# Supported SPARC LLM providers and the ALTK validating-client registry id each +# maps to. All entries are *validating* clients, which SPARC requires. +# +# watsonx (default) and ollama use ALTK's provider-native LiteLLM clients. +# Everything else — openai, azure (Azure OpenAI), and the generic "litellm" +# escape hatch — uses ALTK's generic LiteLLM client (litellm.output_val), which +# routes by the model string (e.g. "gpt-4o-mini", "azure/", +# "anthropic/claude-3-5-sonnet", "gemini/gemini-1.5-pro", "bedrock/..."). This +# means ANY provider LiteLLM/ALTK supports works via config alone — set +# SPARC_LLM_PROVIDER=litellm, SPARC_MODEL=, and pass any extra +# client kwargs (api_base, api_version, api_key, ...) via SPARC_LLM_KWARGS_JSON. +# For full control you can also set SPARC_LLM_REGISTRY_ID to any ALTK registry id. +PROVIDER_REGISTRY_IDS: dict[str, str] = { + "watsonx": "litellm.watsonx.output_val", + "ollama": "litellm.ollama.output_val", + "openai": "litellm.output_val", + "azure": "litellm.output_val", + "litellm": "litellm.output_val", +} + +# Per-provider default model id (empty → SPARC_MODEL is required). +PROVIDER_DEFAULT_MODELS: dict[str, str] = { + "watsonx": "mistral-large-2512", + "ollama": "llama3.2:3b", + "openai": "gpt-4o-mini", + "azure": "", + "litellm": "", +} + +# SPARC reflection tracks exposed via configuration. Mapped to altk Track enum +# lazily in providers.py to avoid importing heavy altk modules at settings time. +SUPPORTED_TRACKS = {"fast_track", "slow_track", "syntax", "spec_free", "transformations_only"} + + +def _truthy(value: str) -> bool: + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _int_env(name: str, default: int) -> int: + raw = os.getenv(name, "").strip() + if not raw: + return default + try: + return int(raw) + except ValueError: + return default + + +@dataclass(frozen=True) +class Settings: + """Immutable, validated view of the service configuration.""" + + provider: str = "watsonx" + model: str = "mistral-large-2512" + track: str = "fast_track" + + # SPARC execution tuning. + llm_timeout_seconds: int = 120 + retries: int = 3 + max_parallel: int = 2 + include_raw_response: bool = True + + # Watsonx credentials (provider == "watsonx"). + wx_api_key: str = "" + wx_project_id: str = "" + wx_url: str = "https://us-south.ml.cloud.ibm.com" + + # Ollama (provider == "ollama"). + ollama_base_url: str = "http://localhost:11434" + ollama_api_key: str = "ollama" + + # OpenAI (provider == "openai"). + openai_api_key: str = "" + openai_base_url: str = "" + + # Generic LiteLLM client kwargs (provider in {openai, azure, litellm}), e.g. + # {"api_base": "...", "api_version": "...", "api_key": "..."}. Parsed from + # SPARC_LLM_KWARGS_JSON. Passed straight to the ALTK litellm client. + llm_kwargs: dict = field(default_factory=dict) + # Optional explicit ALTK registry id override (advanced; any ALTK client). + llm_registry_id: str = "" + + # HTTP server. + host: str = "0.0.0.0" + port: int = 8090 + + # Validation errors collected at load time (provider creds missing, etc.). + errors: tuple[str, ...] = field(default_factory=tuple) + + @property + def registry_id(self) -> str: + return self.llm_registry_id or PROVIDER_REGISTRY_IDS[self.provider] + + def credentials_present(self) -> bool: + return not self.errors + + @classmethod + def from_env(cls) -> "Settings": + provider = os.getenv("SPARC_LLM_PROVIDER", "watsonx").strip().lower() + errors: list[str] = [] + if provider not in PROVIDER_REGISTRY_IDS: + errors.append( + f"unsupported SPARC_LLM_PROVIDER={provider!r}; expected one of {sorted(PROVIDER_REGISTRY_IDS)}" + ) + provider = "watsonx" + + track = os.getenv("SPARC_TRACK", "fast_track").strip().lower() + if track not in SUPPORTED_TRACKS: + errors.append(f"unsupported SPARC_TRACK={track!r}; expected one of {sorted(SUPPORTED_TRACKS)}") + track = "fast_track" + + wx_api_key = os.getenv("WX_API_KEY") or os.getenv("WATSONX_API_KEY") or "" + wx_project_id = os.getenv("WX_PROJECT_ID") or os.getenv("WATSONX_PROJECT_ID") or "" + wx_url = os.getenv("WX_URL") or os.getenv("WATSONX_URL") or "https://us-south.ml.cloud.ibm.com" + + model = os.getenv("SPARC_MODEL") or os.getenv("WX_MODEL_ID") or PROVIDER_DEFAULT_MODELS.get(provider, "") + + # Optional generic LiteLLM client kwargs (azure/openai/litellm). + llm_kwargs: dict = {} + raw_kwargs = os.getenv("SPARC_LLM_KWARGS_JSON", "").strip() + if raw_kwargs: + try: + parsed = json.loads(raw_kwargs) + if isinstance(parsed, dict): + llm_kwargs = parsed + else: + errors.append("SPARC_LLM_KWARGS_JSON must be a JSON object") + except ValueError as exc: + errors.append(f"SPARC_LLM_KWARGS_JSON is not valid JSON: {exc}") + + # Provider-specific credential / config validation. + if provider == "watsonx" and (not wx_api_key or not wx_project_id): + errors.append("provider=watsonx requires WX_API_KEY and WX_PROJECT_ID") + if provider == "openai" and not (os.getenv("OPENAI_API_KEY") or "api_key" in llm_kwargs): + errors.append("provider=openai requires OPENAI_API_KEY (or api_key in SPARC_LLM_KWARGS_JSON)") + if provider in ("azure", "litellm") and not model: + errors.append( + f"provider={provider} requires SPARC_MODEL (e.g. azure/ or anthropic/claude-3-5-sonnet)" + ) + + return cls( + provider=provider, + model=model, + track=track, + llm_timeout_seconds=_int_env("SPARC_LLM_TIMEOUT", 120), + retries=_int_env("SPARC_RETRIES", 3), + max_parallel=_int_env("SPARC_MAX_PARALLEL", 2), + include_raw_response=_truthy(os.getenv("SPARC_INCLUDE_RAW_RESPONSE", "true")), + wx_api_key=wx_api_key, + wx_project_id=wx_project_id, + wx_url=wx_url, + ollama_base_url=os.getenv("OLLAMA_BASE_URL", "http://localhost:11434"), + ollama_api_key=os.getenv("OLLAMA_API_KEY", "ollama"), + openai_api_key=os.getenv("OPENAI_API_KEY", ""), + openai_base_url=os.getenv("OPENAI_BASE_URL", ""), + llm_kwargs=llm_kwargs, + llm_registry_id=os.getenv("SPARC_LLM_REGISTRY_ID", "").strip(), + host=os.getenv("HOST", "0.0.0.0"), + port=_int_env("PORT", 8090), + errors=tuple(errors), + ) diff --git a/authbridge/sparc-service/tests/conftest.py b/authbridge/sparc-service/tests/conftest.py new file mode 100644 index 000000000..59d6e1d23 --- /dev/null +++ b/authbridge/sparc-service/tests/conftest.py @@ -0,0 +1,68 @@ +"""Shared test fixtures. + +The unit tests never touch the network or LLM credentials: they inject a fake +SPARC component (built from real altk result types) into the engine. +""" + +from __future__ import annotations + +import os +from types import SimpleNamespace +from typing import Any + +import pytest +from sparc_service.engine import ReflectionEngine +from sparc_service.settings import Settings + + +def make_reflection_output( + decision: str, + issues: list[dict[str, Any]] | None = None, + overall_avg_score: float | None = None, + execution_time_ms: float = 12.3, +) -> Any: + """Build a real altk reflection output object for fakes.""" + from altk.pre_tool.core import ( + SPARCReflectionDecision, + SPARCReflectionIssue, + SPARCReflectionResult, + ) + + issue_objs = [SPARCReflectionIssue(**i) for i in (issues or [])] + reflection = SPARCReflectionResult(decision=SPARCReflectionDecision(decision), issues=issue_objs) + raw_pipeline = {} + if overall_avg_score is not None: + raw_pipeline["overall_avg_score"] = overall_avg_score + return SimpleNamespace( + output=SimpleNamespace( + reflection_result=reflection, + execution_time_ms=execution_time_ms, + raw_pipeline_result=raw_pipeline, + ) + ) + + +class FakeComponent: + """Stands in for SPARCReflectionComponent; returns a canned verdict.""" + + def __init__(self, output: Any) -> None: + self._output = output + self.calls: list[Any] = [] + + def process(self, run_input: Any, phase: Any) -> Any: + self.calls.append(run_input) + return self._output + + +def make_engine(output: Any, settings: Settings | None = None) -> ReflectionEngine: + settings = settings or Settings(provider="watsonx", wx_api_key="k", wx_project_id="p") + component = FakeComponent(output) + return ReflectionEngine(settings, component_factory=lambda _track: component) + + +@pytest.fixture +def clean_env(monkeypatch): + for key in list(os.environ): + if key.startswith(("SPARC_", "WX_", "WATSONX_", "OLLAMA_", "OPENAI_")): + monkeypatch.delenv(key, raising=False) + yield monkeypatch diff --git a/authbridge/sparc-service/tests/test_api.py b/authbridge/sparc-service/tests/test_api.py new file mode 100644 index 000000000..f33a3598b --- /dev/null +++ b/authbridge/sparc-service/tests/test_api.py @@ -0,0 +1,110 @@ +"""HTTP surface tests via FastAPI TestClient with an injected fake engine.""" + +from __future__ import annotations + +import json + +from fastapi.testclient import TestClient +from sparc_service.api import create_app +from sparc_service.settings import Settings + +from tests.conftest import make_engine, make_reflection_output + +REQUEST_BODY = { + "messages": [{"role": "user", "content": "Refund transaction TX482."}], + "tool_specs": [ + { + "type": "function", + "function": { + "name": "get_transaction", + "parameters": { + "type": "object", + "properties": {"transaction_id": {"type": "string"}}, + "required": ["transaction_id"], + }, + }, + } + ], + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "get_transaction", "arguments": json.dumps({"transaction_id": "TX4821"})}, + } + ], + "session_id": "s1", +} + + +def _client(output) -> TestClient: + return TestClient(create_app(engine=make_engine(output))) + + +def test_reflect_reject(): + output = make_reflection_output( + "reject", + issues=[ + {"issue_type": "semantic_function", "metric_name": "m", "explanation": "no refund tool", "correction": None} + ], + overall_avg_score=2.0, + ) + resp = _client(output).post("/reflect", json=REQUEST_BODY) + assert resp.status_code == 200 + body = resp.json() + assert body["decision"] == "reject" + assert body["overall_avg_score"] == 2.0 + assert body["issues"][0]["explanation"] == "no refund tool" + + +def test_reflect_requires_tool_calls(): + bad = dict(REQUEST_BODY, tool_calls=[]) + resp = _client(make_reflection_output("approve")).post("/reflect", json=bad) + assert resp.status_code == 422 # pydantic validation: min_length=1 + + +def test_healthz_reports_provider(): + resp = _client(make_reflection_output("approve")).get("/healthz") + assert resp.status_code == 200 + assert resp.json()["provider"] == "watsonx" + + +def test_readyz_ok_with_buildable_component(): + resp = _client(make_reflection_output("approve")).get("/readyz") + assert resp.status_code == 200 + assert resp.json()["status"] == "ready" + + +def test_readyz_503_when_config_invalid(): + # Settings with a recorded error -> not ready, never builds component. + settings = Settings(provider="watsonx", errors=("provider=watsonx requires WX_API_KEY and WX_PROJECT_ID",)) + engine = make_engine(make_reflection_output("approve"), settings) + client = TestClient(create_app(engine=engine)) + resp = client.get("/readyz") + assert resp.status_code == 503 + + +def test_reflect_502_hides_backend_error(): + # Provider exception text can embed endpoints/credentials, so /reflect must + # never echo it — it returns a stable message and logs the detail instead. + class BoomEngine: + settings = Settings(provider="watsonx", wx_api_key="k", wx_project_id="p") + + def reflect(self, request): + raise RuntimeError("watsonx exploded with key sk-secret-123") + + def ready(self): + return True, None + + client = TestClient(create_app(engine=BoomEngine())) + resp = client.post("/reflect", json=REQUEST_BODY) + assert resp.status_code == 502 + body = json.dumps(resp.json()) + assert "sk-secret-123" not in body + assert "watsonx exploded" not in body + assert resp.json()["detail"]["error"] == "reflection failed" + + +def test_reflect_400_on_unsupported_track(): + bad = dict(REQUEST_BODY, track="not_a_track") + resp = _client(make_reflection_output("approve")).post("/reflect", json=bad) + assert resp.status_code == 400 diff --git a/authbridge/sparc-service/tests/test_engine.py b/authbridge/sparc-service/tests/test_engine.py new file mode 100644 index 000000000..c0a61de7c --- /dev/null +++ b/authbridge/sparc-service/tests/test_engine.py @@ -0,0 +1,115 @@ +"""Engine IO mapping, using a fake SPARC component (no network).""" + +from __future__ import annotations + +import json + +from sparc_service.models import ReflectRequest + +from tests.conftest import make_engine, make_reflection_output + +FINANCE_TOOL_SPECS = [ + { + "type": "function", + "function": { + "name": "get_transaction", + "description": "Fetch transaction details by transaction id.", + "parameters": { + "type": "object", + "properties": {"transaction_id": {"type": "string"}}, + "required": ["transaction_id"], + }, + }, + } +] + + +def _request(args: dict) -> ReflectRequest: + return ReflectRequest( + messages=[{"role": "user", "content": "Refund transaction TX482."}], + tool_specs=FINANCE_TOOL_SPECS, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_transaction", "arguments": json.dumps(args)}, + } + ], + session_id="s1", + ) + + +def test_approve_maps_cleanly(): + engine = make_engine(make_reflection_output("approve", overall_avg_score=4.5)) + resp = engine.reflect(_request({"transaction_id": "TX4827"})) + assert resp.decision == "approve" + assert resp.issues == [] + assert resp.overall_avg_score == 4.5 + assert resp.execution_time_ms == 12.3 + + +def test_reject_carries_issue_and_correction(): + output = make_reflection_output( + "reject", + issues=[ + { + "issue_type": "semantic_function", + "metric_name": "function_selection_appropriateness", + "explanation": "get_transaction does not process refunds.", + "correction": {"corrected_function_name": "no_function"}, + } + ], + overall_avg_score=2.0, + ) + engine = make_engine(output) + resp = engine.reflect(_request({"transaction_id": "TX4821"})) + assert resp.decision == "reject" + assert len(resp.issues) == 1 + issue = resp.issues[0] + assert issue.metric_name == "function_selection_appropriateness" + assert "refund" in issue.explanation + assert issue.correction["corrected_function_name"] == "no_function" + assert resp.overall_avg_score == 2.0 + + +def test_error_decision_passthrough(): + engine = make_engine(make_reflection_output("error")) + resp = engine.reflect(_request({"transaction_id": "TX4821"})) + assert resp.decision == "error" + + +def test_raw_pipeline_suppressed_when_disabled(): + from sparc_service.settings import Settings + + settings = Settings(provider="watsonx", wx_api_key="k", wx_project_id="p", include_raw_response=False) + engine = make_engine(make_reflection_output("approve", overall_avg_score=5.0), settings) + resp = engine.reflect(_request({"transaction_id": "TX4827"})) + # Score is still surfaced, but the verbose raw pipeline is withheld. + assert resp.overall_avg_score == 5.0 + assert resp.raw_pipeline_result is None + + +def test_run_input_receives_request_fields(): + engine = make_engine(make_reflection_output("approve")) + engine.reflect(_request({"transaction_id": "TX4827"})) + # The fake records the run_input the engine handed to SPARC. + component = next(iter(engine._components.values())) # noqa: SLF001 - test introspection + assert len(component.calls) == 1 + run_input = component.calls[0] + # All three SPARC inputs reach the component unchanged. + assert run_input.messages == [{"role": "user", "content": "Refund transaction TX482."}] + assert run_input.tool_specs == FINANCE_TOOL_SPECS + assert run_input.tool_calls[0]["function"]["name"] == "get_transaction" + assert run_input.tool_calls[0]["function"]["arguments"] == json.dumps({"transaction_id": "TX4827"}) + + +def test_unsupported_track_rejected(): + engine = make_engine(make_reflection_output("approve")) + request = _request({"transaction_id": "TX4827"}) + request.track = "made_up_track" + try: + engine.reflect(request) + except ValueError as exc: + assert "unsupported track" in str(exc) + else: + raise AssertionError("expected ValueError for an unsupported track") diff --git a/authbridge/sparc-service/tests/test_providers.py b/authbridge/sparc-service/tests/test_providers.py new file mode 100644 index 000000000..89199a64d --- /dev/null +++ b/authbridge/sparc-service/tests/test_providers.py @@ -0,0 +1,52 @@ +"""LLM-client construction: provider-native vs explicit registry override. + +These tests stub ALTK's ``get_llm`` so they exercise the kwarg-selection logic +in ``build_llm_client`` without authenticating against any real provider. +""" + +from __future__ import annotations + +from typing import Any + +import altk.core.llm as llm_mod +from sparc_service.providers import build_llm_client +from sparc_service.settings import Settings + + +def _capture(monkeypatch) -> dict[str, Any]: + captured: dict[str, Any] = {} + + class FakeClient: + def __init__(self, **kwargs: Any) -> None: + captured.update(kwargs) + + monkeypatch.setattr(llm_mod, "get_llm", lambda _registry_id: FakeClient) + return captured + + +def test_native_watsonx_passes_provider_kwargs(monkeypatch): + captured = _capture(monkeypatch) + s = Settings(provider="watsonx", wx_api_key="k", wx_project_id="p", model="mistral-large-2512") + build_llm_client(s) + assert captured["project_id"] == "p" + assert captured["model_name"] == "mistral-large-2512" + + +def test_registry_override_skips_provider_kwargs(monkeypatch): + # With an explicit registry override, the watsonx-native project_id/api_base + # kwargs must NOT be passed (they'd break a generic LiteLLM client). Only the + # generic kwargs from llm_kwargs reach the constructor. + captured = _capture(monkeypatch) + s = Settings( + provider="watsonx", + wx_api_key="k", + wx_project_id="p", + model="gpt-4o-mini", + llm_registry_id="litellm.output_val", + llm_kwargs={"api_key": "sk-x"}, + ) + build_llm_client(s) + assert "project_id" not in captured + assert "api_base" not in captured + assert captured["model_name"] == "gpt-4o-mini" + assert captured["api_key"] == "sk-x" diff --git a/authbridge/sparc-service/tests/test_settings.py b/authbridge/sparc-service/tests/test_settings.py new file mode 100644 index 000000000..0d2ef0dd3 --- /dev/null +++ b/authbridge/sparc-service/tests/test_settings.py @@ -0,0 +1,105 @@ +"""Settings parsing and provider-credential validation.""" + +from __future__ import annotations + +from sparc_service.settings import Settings + + +def test_defaults_to_watsonx(clean_env): + clean_env.setenv("WX_API_KEY", "key") + clean_env.setenv("WX_PROJECT_ID", "proj") + s = Settings.from_env() + assert s.provider == "watsonx" + assert s.registry_id == "litellm.watsonx.output_val" + assert s.model == "mistral-large-2512" + assert s.credentials_present() + + +def test_watsonx_missing_creds_records_error(clean_env): + s = Settings.from_env() + assert not s.credentials_present() + assert any("WX_API_KEY" in e for e in s.errors) + + +def test_watsonx_fallback_env_names(clean_env): + clean_env.setenv("WATSONX_API_KEY", "key") + clean_env.setenv("WATSONX_PROJECT_ID", "proj") + s = Settings.from_env() + assert s.wx_api_key == "key" + assert s.wx_project_id == "proj" + assert s.credentials_present() + + +def test_ollama_provider_needs_no_watsonx(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "ollama") + s = Settings.from_env() + assert s.provider == "ollama" + assert s.registry_id == "litellm.ollama.output_val" + assert s.model == "llama3.2:3b" + assert s.credentials_present() + + +def test_openai_provider_requires_key(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "openai") + s = Settings.from_env() + assert not s.credentials_present() + clean_env.setenv("OPENAI_API_KEY", "sk-x") + assert Settings.from_env().credentials_present() + + +def test_unsupported_provider_records_error(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "bogus") + s = Settings.from_env() + assert any("unsupported SPARC_LLM_PROVIDER" in e for e in s.errors) + + +def test_model_override(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "ollama") + clean_env.setenv("SPARC_MODEL", "qwen2.5:7b") + assert Settings.from_env().model == "qwen2.5:7b" + + +def test_generic_litellm_provider(clean_env): + # The generic escape hatch supports any LiteLLM model (anthropic/gemini/...). + clean_env.setenv("SPARC_LLM_PROVIDER", "litellm") + clean_env.setenv("SPARC_MODEL", "anthropic/claude-3-5-sonnet") + clean_env.setenv("SPARC_LLM_KWARGS_JSON", '{"api_key": "sk-ant-x"}') + s = Settings.from_env() + assert s.registry_id == "litellm.output_val" + assert s.model == "anthropic/claude-3-5-sonnet" + assert s.llm_kwargs == {"api_key": "sk-ant-x"} + assert s.credentials_present() + + +def test_litellm_provider_requires_model(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "litellm") + s = Settings.from_env() + assert any("requires SPARC_MODEL" in e for e in s.errors) + + +def test_azure_provider(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "azure") + clean_env.setenv("SPARC_MODEL", "azure/my-deployment") + clean_env.setenv( + "SPARC_LLM_KWARGS_JSON", + '{"api_base": "https://x.openai.azure.com", "api_version": "2024-06-01", "api_key": "k"}', + ) + s = Settings.from_env() + assert s.registry_id == "litellm.output_val" + assert s.llm_kwargs["api_version"] == "2024-06-01" + assert s.credentials_present() + + +def test_invalid_kwargs_json_records_error(clean_env): + clean_env.setenv("SPARC_LLM_PROVIDER", "litellm") + clean_env.setenv("SPARC_MODEL", "anthropic/claude-3-5-sonnet") + clean_env.setenv("SPARC_LLM_KWARGS_JSON", "{not json}") + s = Settings.from_env() + assert any("SPARC_LLM_KWARGS_JSON" in e for e in s.errors) + + +def test_registry_id_override(clean_env): + clean_env.setenv("WX_API_KEY", "key") + clean_env.setenv("WX_PROJECT_ID", "proj") + clean_env.setenv("SPARC_LLM_REGISTRY_ID", "azure_openai.sync.output_val") + assert Settings.from_env().registry_id == "azure_openai.sync.output_val" diff --git a/authbridge/sparc-service/tests/test_watsonx_integration.py b/authbridge/sparc-service/tests/test_watsonx_integration.py new file mode 100644 index 000000000..cd20224d4 --- /dev/null +++ b/authbridge/sparc-service/tests/test_watsonx_integration.py @@ -0,0 +1,86 @@ +"""Opt-in integration test against real watsonx. + +Skipped unless RUN_WATSONX_TESTS=1 and WX_* credentials are present. Validates +the canonical finance scenario end-to-end through the real SPARC engine: +an ungrounded/inappropriate tool call must be rejected. + +Run with: + RUN_WATSONX_TESTS=1 pytest tests/test_watsonx_integration.py -v +""" + +from __future__ import annotations + +import json +import os + +import pytest +from sparc_service.engine import ReflectionEngine +from sparc_service.models import ReflectRequest +from sparc_service.settings import Settings + +pytestmark = pytest.mark.skipif( + os.getenv("RUN_WATSONX_TESTS") != "1" or not (os.getenv("WX_API_KEY") or os.getenv("WATSONX_API_KEY")), + reason="set RUN_WATSONX_TESTS=1 and WX_* creds to run watsonx integration tests", +) + +TOOL_SPECS = [ + { + "type": "function", + "function": { + "name": "get_transaction", + "description": "Fetch transaction details by transaction id.", + "parameters": { + "type": "object", + "properties": {"transaction_id": {"type": "string", "description": "Exact transaction identifier."}}, + "required": ["transaction_id"], + }, + }, + } +] + + +def _engine() -> ReflectionEngine: + return ReflectionEngine(Settings.from_env()) + + +def test_ungrounded_partial_id_is_rejected(): + """User gives partial 'TX482'; model hallucinates 'TX4821' -> reject.""" + req = ReflectRequest( + messages=[ + {"role": "user", "content": "Refund transaction TX482 because it was a duplicate charge."}, + {"role": "assistant", "content": "I will process the refund."}, + ], + tool_specs=TOOL_SPECS, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_transaction", "arguments": json.dumps({"transaction_id": "TX4821"})}, + } + ], + session_id="itest-reject", + ) + resp = _engine().reflect(req) + assert resp.decision in {"reject", "error"} + assert resp.decision == "reject", f"expected reject, got {resp.decision}: {resp.issues}" + assert resp.issues, "reject should carry at least one issue" + + +def test_grounded_full_id_is_approved(): + """Once the user provides the exact id, the same call is grounded -> approve.""" + req = ReflectRequest( + messages=[ + {"role": "user", "content": "Look up transaction TX4827 for me."}, + ], + tool_specs=TOOL_SPECS, + tool_calls=[ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_transaction", "arguments": json.dumps({"transaction_id": "TX4827"})}, + } + ], + session_id="itest-approve", + ) + resp = _engine().reflect(req) + assert resp.decision == "approve", f"expected approve, got {resp.decision}: {resp.issues}"