feat(pipeline): Config hot-reload via fsnotify + atomic pipeline swap

authbridge/CLAUDE.md

@@ -112,7 +112,7 @@ with protocol-specific listeners in `cmd/authbridge/listener/`:
 
 **Configuration loading:**
 - YAML config with `${ENV_VAR}` expansion, mode presets, and startup validation.
-- Plugin settings are local to each plugin under `pipeline.*.plugins[].config`; the runtime YAML itself only carries `mode`, `listener`, `session`, `stats`, and the pipeline composition. See [`cmd/authbridge/README.md`](cmd/authbridge/README.md) for the per-mode YAML shape and [`authlib/plugins/CONVENTIONS.md`](authlib/plugins/CONVENTIONS.md) for the per-plugin decode pattern.
+- Plugin settings are local to each plugin under `pipeline.*.plugins[].config`; the runtime YAML itself only carries `mode`, `listener`, `session`, `stats`, and the pipeline composition. See [`cmd/authbridge/README.md`](cmd/authbridge/README.md) for the per-mode YAML shape and [`docs/plugin-reference.md`](docs/plugin-reference.md) for the per-plugin decode pattern.
 - The operator-supplied env vars (`KEYCLOAK_URL`, `KEYCLOAK_REALM`, `TOKEN_URL`, `ISSUER`, `DEFAULT_OUTBOUND_POLICY`, `CLIENT_ID`) are consumed by the default `authbridge-combined.yaml` via `${VAR}` expansion — they land inside the appropriate plugin's `config:` block rather than a top-level section.
 - `jwt-validation` derives `jwks_url` from `issuer` when omitted (appends `/protocol/openid-connect/certs`).
 - `token-exchange` derives `token_url` from `keycloak_url + keycloak_realm` when omitted (Keycloak convention).
@@ -127,8 +127,8 @@ with protocol-specific listeners in `cmd/authbridge/listener/`:
 - `authlib/routing/` -- Host-to-audience route resolver (used internally by `token-exchange` plugin)
 - `authlib/auth/` -- `HandleInbound` + `HandleOutbound` composition; each plugin instance constructs its own `auth.Auth` from its own local config
 - `authlib/config/` -- Mode presets, YAML config loader, credential-file waiters, top-level (mode + listener + session) validation
-- `authlib/pipeline/` -- Plugin interface + lifecycle (`Configurable`, `Initializer`, `Shutdowner`); see [`authlib/pipeline/README.md`](authlib/pipeline/README.md)
-- `authlib/plugins/` -- The concrete plugins + registry; see [`authlib/plugins/CONVENTIONS.md`](authlib/plugins/CONVENTIONS.md) for the per-plugin config convention
+- `authlib/pipeline/` -- Plugin interface + lifecycle (`Configurable`, `Initializer`, `Shutdowner`); see [`docs/framework-architecture.md`](docs/framework-architecture.md)
+- `authlib/plugins/` -- The concrete plugins + registry; see [`docs/plugin-reference.md`](docs/plugin-reference.md) for the per-plugin config convention
 
 ### init-iptables.sh
 
@@ -289,7 +289,7 @@ kubectl apply -f k8s/auth-target-deployment-webhook.yaml     # Target service
 | 15123 | Envoy | TCP | Outbound listener (iptables redirects app traffic here) |
 | 15124 | Envoy | TCP | Inbound listener (iptables redirects incoming traffic here) |
 | 9090 | authbridge | gRPC | Ext-proc server (called by Envoy) |
-| 9093 | authbridge | HTTP | Stats + config inspection (`/stats`, `/config`) |
+| 9093 | authbridge | HTTP | Stats + config inspection (`/stats`, `/config`, `/reload/status`) |
 | 9094 | authbridge | HTTP | Session events API (JSON snapshots + SSE stream) |
 | 9901 | Envoy | HTTP | Admin interface (bound to 127.0.0.1) |
 | 8080 | auth-proxy | HTTP | Example app (NOT part of sidecar) |
@@ -340,7 +340,7 @@ Every event on `/v1/sessions/{id}` and `/v1/events` carries:
 - `at`, `direction`, `phase` — when, which side, what stage. `phase` is one of `"request"`, `"response"`, or `"denied"` (terminal denial from a pipeline plugin — typically a jwt-validation failure).
 - `a2a` / `mcp` / `inference` — protocol parser payloads (one at most).
 - `invocations` — per-plugin invocation records for every plugin that ran on the pipeline pass. Structured as `{inbound: [...], outbound: [...]}`; each entry carries `plugin`, `action` (one of 5 values — see below), `reason` (machine-stable code), and optional plugin-specific context (expected issuer, target audience, cache-hit flag, path, etc.). abctl renders one row per invocation, so operators see an explicit per-plugin timeline.
-- `plugins` — escape-hatch map for plugin-specific observability. Keys are plugin names; values are the raw JSON each plugin emitted. Unknown plugins render as opaque JSON in abctl. See `authlib/plugins/CONVENTIONS.md` "Emitting session events" for the producer contract.
+- `plugins` — escape-hatch map for plugin-specific observability. Keys are plugin names; values are the raw JSON each plugin emitted. Unknown plugins render as opaque JSON in abctl. See [`docs/plugin-reference.md`](docs/plugin-reference.md#emitting-session-events) for the producer contract.
 - `identity`, `host`, `statusCode`, `error`, `durationMs` — request-level context.
 
 ### Invocation action vocabulary
@@ -357,6 +357,8 @@ Every plugin emits one of these 5 action values per invocation, so operators can
 
 Use `reason` to discriminate within an action — e.g. `skip/path_bypass` vs `skip/no_matching_route` tell different stories at the detail-pane level but both scan as "skip" in the at-a-glance timeline.
 
+> **Producer-side contract:** the authoritative definition of the 5-value vocabulary, the `Invocation` struct fields, and which diagnostic fields each plugin type populates lives in [`docs/plugin-reference.md`](docs/plugin-reference.md#emitting-session-events). Edit that file when the vocabulary changes; this table is the consumer-side summary.
+
 ### Gotcha: denied requests
 
 Rejected requests (401 / 503) land as `phase: "denied"` events in `/v1/sessions` when at least one pipeline plugin appended an Invocation before rejecting. If you're debugging an unauthorized-access pattern, the default-session bucket (`GET /v1/sessions/default`) is where denial events aggregate.
@@ -365,6 +367,27 @@ Rejected requests (401 / 503) land as `phase: "denied"` events in `/v1/sessions`
 
 Set `session.enabled: false` in the runtime config to turn off the store (and implicitly the API). Setting `listener.session_api_addr: ""` alone is not currently supported as a selective disable — the preset refills it; if you need store-on-API-off, raise an issue.
 
+## Config Hot-Reload (`:9093/reload/status`)
+
+The authbridge binary watches its config file (`/etc/authbridge/config.yaml`) via `authlib/reloader`. When the ConfigMap changes, kubelet syncs the new content into the mount (~60s), the watcher detects it, and the binary rebuilds + atomically swaps the plugin pipelines without a pod restart. In-flight requests finish on the previous pipeline; new requests go to the new one.
+
+**What reloads:** any plugin list change (add/remove/reorder) and any plugin `config:` subtree edit.
+
+**What doesn't reload (pod restart required):** `mode`, any `listener.*` address, and the session store parameters (`session.ttl`, `session.max_events`, `session.max_sessions`).
+
+**Bad YAML stays safe:** if Load/Validate/Build/Start fails, the active pipeline keeps serving and the error is exposed on `/reload/status` with `reloads_failed` incremented. The pod never goes unhealthy from a bad edit.
+
+Operator workflow:
+
+```sh
+kubectl edit configmap authbridge-config-<agent> -n <ns>
+kubectl port-forward -n <ns> deploy/<agent> 9093:9093 &
+curl http://localhost:9093/reload/status  # last_success, reloads_ok, active_config_sha256
+curl http://localhost:9093/config         # now-active config (ConfigProvider closure)
+```
+
+See [`docs/framework-architecture.md`](docs/framework-architecture.md#9-config-hot-reload) §9 for the reload lifecycle, debounce / symlink-swap handling, and the drain-window behavior.
+
 ## Code Conventions
 
 ### Go (authlib, cmd/authbridge, demo-app)

authbridge/authlib/README.md

@@ -14,15 +14,15 @@ A pure Go library providing reusable building blocks for JWT validation, OAuth 2
 | `routing/` | Host-to-audience router with glob pattern matching |
 | `auth/` | Composition layer: `HandleInbound` + `HandleOutbound` — used internally by `jwt-validation` and `token-exchange` plugins |
 | `config/` | YAML config loader, mode presets, credential-file waiters, top-level validation |
-| `pipeline/` | Plugin pipeline + lifecycle (`Configurable`, `Initializer`, `Shutdowner`) — see [pipeline/README.md](./pipeline/README.md) |
+| `pipeline/` | Plugin pipeline + lifecycle (`Configurable`, `Initializer`, `Shutdowner`) — see [docs/framework-architecture.md](../docs/framework-architecture.md) |
 | `session/` | In-memory session store + `SessionSummary` aggregation, backing the `:9094` API |
 | `sessionapi/` | HTTP API (`/v1/sessions`, `/v1/events` SSE, `/v1/pipeline`) exposing the session store |
-| `plugins/` | Built-in plugins: `jwt-validation`, `token-exchange`, `a2a-parser`, `mcp-parser`, `inference-parser`. See [plugins/CONVENTIONS.md](./plugins/CONVENTIONS.md) for the per-plugin config convention |
+| `plugins/` | Built-in plugins: `jwt-validation`, `token-exchange`, `a2a-parser`, `mcp-parser`, `inference-parser`. See [docs/plugin-reference.md](../docs/plugin-reference.md) for the per-plugin config convention |
 | `observe/` | OTEL + metrics helpers |
 
 ## Usage
 
-Plugins own their own configuration and construct any `auth.Auth` they need internally from their per-plugin config (see [plugins/CONVENTIONS.md](./plugins/CONVENTIONS.md)). Host processes (cmd/authbridge) just load the YAML, call `plugins.Build`, run the pipelines, and let each plugin handle its own dependencies.
+Plugins own their own configuration and construct any `auth.Auth` they need internally from their per-plugin config (see [docs/plugin-reference.md](../docs/plugin-reference.md)). Host processes (cmd/authbridge) just load the YAML, call `plugins.Build`, run the pipelines, and let each plugin handle its own dependencies.
 
 ```go
 import (

authbridge/authlib/config/config.go

@@ -14,8 +14,8 @@ import (
 //
 // Plugin-specific settings (inbound JWT validation, outbound token
 // exchange, identity, bypass paths, routes) live inside their
-// respective entries under Pipeline.* now — see the plugin README at
-// authbridge/authlib/plugins/CONVENTIONS.md for how each plugin
+// respective entries under Pipeline.* now — see the plugin reference at
+// authbridge/docs/plugin-reference.md for how each plugin
 // declares its own config schema and defaults.
 type Config struct {
 	Mode     string         `yaml:"mode" json:"mode"` // "envoy-sidecar", "waypoint", "proxy-sidecar"
@@ -70,7 +70,7 @@ type PipelineStageConfig struct {
 // full form ({name, id, config}). The short form keeps existing pipeline
 // configs parsing unchanged; the long form is what plugins that
 // implement pipeline.Configurable actually need. See
-// authbridge/authlib/plugins/CONVENTIONS.md for the convention plugins
+// authbridge/docs/plugin-reference.md for the convention plugins
 // follow when decoding Config.
 //
 // Config is captured as a raw subtree via json.RawMessage so the plugin

authbridge/authlib/config/presets.go

@@ -2,7 +2,7 @@ package config
 
 // ApplyPreset fills in mode-specific defaults for listener addresses.
 // Plugin-specific defaults live inside each plugin's Configure (see
-// authbridge/authlib/plugins/CONVENTIONS.md); the runtime config no
+// authbridge/docs/plugin-reference.md); the runtime config no
 // longer shapes plugin behavior.
 func ApplyPreset(cfg *Config) {
 	switch cfg.Mode {

authbridge/authlib/config/resolve.go

@@ -1,7 +1,7 @@
 // Package config's resolve.go previously constructed a shared
 // auth.Config + auth.Auth for all plugins to share. With per-plugin
 // configuration, that responsibility moved into each plugin's Configure
-// (see authbridge/authlib/plugins/CONVENTIONS.md), and this file is now
+// (see authbridge/docs/plugin-reference.md), and this file is now
 // just the shared credential-file waiters that multiple plugins need
 // when they share a file path (e.g. /shared/client-id.txt used by both
 // jwt-validation's audience_file and token-exchange's client_id_file).

authbridge/authlib/go.mod

@@ -12,6 +12,7 @@ require (
 
 require (
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.3 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect

authbridge/authlib/go.sum

@@ -3,6 +3,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=

authbridge/authlib/observe/statserver.go

@@ -23,14 +23,43 @@ type StatServer struct {
 // should be cheap (a few map copies).
 type StatsProvider func() *auth.Stats
 
-// NewStatServer builds the stat HTTP server. The statsProvider is
-// invoked per /stats request so per-plugin counters can be aggregated
-// live rather than captured at process start.
-func NewStatServer(addr string, config *config.Config, statsProvider StatsProvider) *StatServer {
+// ConfigProvider returns the currently-active *config.Config per
+// /config request. Used so the endpoint reflects a hot-reload swap
+// performed by authlib/reloader; for setups without hot-reload the
+// host just wraps a captured pointer (`func() *Config { return cfg }`).
+type ConfigProvider func() *config.Config
+
+// Option configures a StatServer at construction time.
+type Option func(*statServerOpts)
+
+type statServerOpts struct {
+	reloadStatus http.Handler
+}
+
+// WithReloadStatus registers a /reload/status handler (typically the
+// Handler returned by an authlib/reloader.Reloader). Omit when hot-
+// reload isn't wired up — the endpoint simply won't exist.
+func WithReloadStatus(h http.Handler) Option {
+	return func(o *statServerOpts) { o.reloadStatus = h }
+}
+
+// NewStatServer builds the stat HTTP server. configProvider is
+// invoked per /config request and statsProvider per /stats request,
+// so both reflect current state rather than a snapshot captured at
+// construction.
+func NewStatServer(addr string, configProvider ConfigProvider, statsProvider StatsProvider, opts ...Option) *StatServer {
+	var o statServerOpts
+	for _, opt := range opts {
+		opt(&o)
+	}
+
 	mux := http.NewServeMux()
 
-	mux.HandleFunc("/config", handleConfigFactory(config))
+	mux.HandleFunc("/config", handleConfigFactory(configProvider))
 	mux.HandleFunc("/stats", handleStatsFactory(statsProvider))
+	if o.reloadStatus != nil {
+		mux.Handle("/reload/status", o.reloadStatus)
+	}
 
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "text/html; charset=utf-8")
@@ -41,6 +70,7 @@ func NewStatServer(addr string, config *config.Config, statsProvider StatsProvid
     <ul>
     <li><a href="/config">Kagenti AuthBridge configuration</a></li>
     <li><a href="/stats">Kagenti AuthBridge statistics</a></li>
+    <li><a href="/reload/status">Config reload status</a></li>
     </ul>
   </body>
 </html>`)
@@ -55,7 +85,7 @@ func NewStatServer(addr string, config *config.Config, statsProvider StatsProvid
 	}
 }
 
-func handleConfigFactory(cfg *config.Config) func(http.ResponseWriter, *http.Request) {
+func handleConfigFactory(provider ConfigProvider) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
 		// Plugin config subtrees are captured verbatim as json.RawMessage
@@ -65,7 +95,7 @@ func handleConfigFactory(cfg *config.Config) func(http.ResponseWriter, *http.Req
 		// so we render the config as-is. If a plugin ever needs to
 		// suppress a known-sensitive field here, it can be added to a
 		// redaction pass in a follow-up.
-		err := json.NewEncoder(w).Encode(cfg)
+		err := json.NewEncoder(w).Encode(provider())
 		if err != nil {
 			slog.Default().Info("Failed to send configuration", "err", err)
 		}

authbridge/authlib/observe/statserver_test.go

@@ -31,7 +31,7 @@ func serveMux(cfg *config.Config, stats *auth.Stats) http.Handler {
 	// what /stats returns. Production callers pass a provider that
 	// merges per-plugin Stats at request time; see main.go's
 	// statsProvider closure.
-	s := NewStatServer(":0", cfg, func() *auth.Stats { return stats })
+	s := NewStatServer(":0", func() *config.Config { return cfg }, func() *auth.Stats { return stats })
 	return s.server.Handler
 }
 
@@ -134,14 +134,14 @@ func TestStatsEndpointValidJSON(t *testing.T) {
 }
 
 func TestNewStatServerSetsAddr(t *testing.T) {
-	s := NewStatServer(":9093", newTestConfig(), func() *auth.Stats { return auth.NewStats() })
+	s := NewStatServer(":9093", func() *config.Config { return newTestConfig() }, func() *auth.Stats { return auth.NewStats() })
 	if s.server.Addr != ":9093" {
 		t.Errorf("server.Addr = %q, want :9093", s.server.Addr)
 	}
 }
 
 func TestNewStatServerCustomAddr(t *testing.T) {
-	s := NewStatServer("127.0.0.1:8888", newTestConfig(), func() *auth.Stats { return auth.NewStats() })
+	s := NewStatServer("127.0.0.1:8888", func() *config.Config { return newTestConfig() }, func() *auth.Stats { return auth.NewStats() })
 	if s.server.Addr != "127.0.0.1:8888" {
 		t.Errorf("server.Addr = %q, want 127.0.0.1:8888", s.server.Addr)
 	}