mcp(feat[middleware,server]): add ReadonlyRetryMiddleware scoped by TAG_READONLY

Adopts FastMCP's ``RetryMiddleware``
(``fastmcp/server/middleware/error_handling.py:136``) but bounded
by the existing safety-tier annotation. The ``_server_cache``
already evicts dead Server instances on ``is_alive() == False``,
but the *first* call after a tmux daemon crash still fails before
the cache learns about it. A single retry on the freshly-cached
Server closes that window without changing tool semantics.

Critical safety constraint: only readonly tools are retried.
Mutating tools like ``send_keys``, ``create_session``,
``split_window`` are NOT idempotent — re-running them on a
transient socket error would silently double side effects, which
is unacceptable. The wrapper ``ReadonlyRetryMiddleware`` reads
``tool.tags`` via ``context.fastmcp_context.fastmcp.get_tool(name)``
(same pattern ``SafetyMiddleware`` uses) and only delegates to
the upstream retry when ``TAG_READONLY`` is present.

Defaults are deliberately small: ``max_retries=1`` keeps audit
log noise minimal (one extra attempt, not three); ``base_delay
0.1s`` / ``max_delay 1.0s`` matches the duration window of a
transient libtmux socket hiccup. ``retry_exceptions=
(libtmux.exc.LibTmuxException,)`` is narrow on purpose — the
upstream default ``(ConnectionError, TimeoutError)`` does NOT
match libtmux's wrapped subprocess errors and would be a silent
no-op.

Wired into the middleware stack between ``AuditMiddleware`` and
``SafetyMiddleware`` so retries are audited once each and
tier-denied tools never reach retry. Updates the existing
``test_server_middleware_stack_order`` to lock in the new
position.

Per-pair regression test in the next commit asserts: readonly
tool retries on ``LibTmuxException``, mutating tool does not
retry, non-libtmux exception does not retry.

Author: tony

src/libtmux_mcp/middleware.py

@@ -26,8 +26,10 @@
 
 from fastmcp.exceptions import ToolError
 from fastmcp.server.middleware import Middleware, MiddlewareContext
+from fastmcp.server.middleware.error_handling import RetryMiddleware
 from fastmcp.server.middleware.response_limiting import ResponseLimitingMiddleware
 from fastmcp.tools.base import ToolResult
+from libtmux import exc as libtmux_exc
 from mcp.types import TextContent
 
 from libtmux_mcp._utils import TAG_DESTRUCTIVE, TAG_MUTATING, TAG_READONLY
@@ -233,6 +235,69 @@ async def on_call_tool(
 #: opted out via ``max_lines=None``.
 DEFAULT_RESPONSE_LIMIT_BYTES = 50_000
 
+
+class ReadonlyRetryMiddleware(Middleware):
+    """Retry transient libtmux failures, but only for readonly tools.
+
+    Wraps fastmcp's :class:`fastmcp.server.middleware.error_handling.RetryMiddleware`
+    so retries are bounded by the safety tier the tool is registered
+    under. Mutating and destructive tools (``send_keys``,
+    ``create_session``, ``kill_server``, …) pass straight through —
+    re-running them on a transient socket error would silently double
+    side effects, which is unacceptable. Readonly tools
+    (``list_sessions``, ``capture_pane``, ``snapshot_pane``, …) are
+    safe to retry because they observe state without mutating it.
+
+    Default retry trigger is :class:`libtmux.exc.LibTmuxException` —
+    libtmux wraps the subprocess failures we actually want to retry
+    (socket EAGAIN, transient connect errors). The fastmcp default
+    ``(ConnectionError, TimeoutError)`` does NOT match these, so the
+    upstream defaults would be a silent no-op.
+
+    Place this in the middleware stack **inside** ``AuditMiddleware``
+    (so retried calls are audited once each) and **outside**
+    ``SafetyMiddleware`` (so tier-denied tools never reach retry).
+    """
+
+    def __init__(
+        self,
+        max_retries: int = 1,
+        base_delay: float = 0.1,
+        max_delay: float = 1.0,
+        backoff_multiplier: float = 2.0,
+        retry_exceptions: tuple[type[Exception], ...] = (libtmux_exc.LibTmuxException,),
+        logger_: logging.Logger | None = None,
+    ) -> None:
+        """Configure the underlying retry policy.
+
+        Defaults are deliberately small. ``max_retries=1`` keeps audit
+        log noise minimal (one retry per failed call, not three). The
+        100 ms / 1 s backoff window matches the expected duration of a
+        transient libtmux socket hiccup — a longer backoff would just
+        delay a real failure without adding meaningful retry headroom.
+        """
+        self._retry = RetryMiddleware(
+            max_retries=max_retries,
+            base_delay=base_delay,
+            max_delay=max_delay,
+            backoff_multiplier=backoff_multiplier,
+            retry_exceptions=retry_exceptions,
+            logger=logger_,
+        )
+
+    async def on_call_tool(
+        self,
+        context: MiddlewareContext,
+        call_next: t.Any,
+    ) -> t.Any:
+        """Delegate to the upstream retry only for tools tagged readonly."""
+        if context.fastmcp_context:
+            tool = await context.fastmcp_context.fastmcp.get_tool(context.message.name)
+            if tool and TAG_READONLY in tool.tags:
+                return await self._retry.on_request(context, call_next)
+        return await call_next(context)
+
+
 #: Header prefixed to a truncated response. Intentionally matches the
 #: format used by the per-tool ``capture_pane`` truncation so clients
 #: see a consistent marker regardless of which layer fired.

src/libtmux_mcp/server.py

@@ -29,6 +29,7 @@
 from libtmux_mcp.middleware import (
     DEFAULT_RESPONSE_LIMIT_BYTES,
     AuditMiddleware,
+    ReadonlyRetryMiddleware,
     SafetyMiddleware,
     TailPreservingResponseLimitingMiddleware,
 )
@@ -221,7 +222,11 @@ def _gc_mcp_buffers(cache: t.Mapping[_ServerCacheKey, Server]) -> None:
     #      Safety) are still logged with outcome=error. Without this
     #      ordering, denied access attempts would silently bypass the
     #      audit log — a security-observability gap.
-    #   5. SafetyMiddleware — innermost gate (fail-closed). Denials
+    #   5. ReadonlyRetryMiddleware — inside Audit so retries are
+    #      audited once each, outside Safety so tier-denied tools
+    #      never reach retry. Only readonly tools are retried;
+    #      mutating/destructive tools pass straight through.
+    #   6. SafetyMiddleware — innermost gate (fail-closed). Denials
     #      never reach the tool, but the audit record above captures
     #      them for forensic review.
     middleware=[
@@ -232,6 +237,7 @@ def _gc_mcp_buffers(cache: t.Mapping[_ServerCacheKey, Server]) -> None:
         ),
         ErrorHandlingMiddleware(transform_errors=True),
         AuditMiddleware(),
+        ReadonlyRetryMiddleware(),
         SafetyMiddleware(max_tier=_safety_level),
     ],
     on_duplicate="error",

tests/test_middleware.py

@@ -326,12 +326,17 @@ def test_server_middleware_stack_order() -> None:
     refactor that swaps Audit and Safety would degrade
     security observability without an obvious test failure, so pin
     the sequence explicitly.
+
+    ReadonlyRetryMiddleware sits between Audit and Safety so retried
+    calls are audited once each (Audit wraps the retry loop) and
+    tier-denied tools never reach retry (Safety stops them first).
     """
     from fastmcp.server.middleware.error_handling import ErrorHandlingMiddleware
     from fastmcp.server.middleware.timing import TimingMiddleware
 
     from libtmux_mcp.middleware import (
         AuditMiddleware,
+        ReadonlyRetryMiddleware,
         SafetyMiddleware,
         TailPreservingResponseLimitingMiddleware,
     )
@@ -341,11 +346,12 @@ def test_server_middleware_stack_order() -> None:
     # FastMCP auto-appends an internal DereferenceRefsMiddleware at the
     # end of the stack; we care about the ordering of the middleware
     # *we* configured. Slice off the suffix before comparing.
-    assert types[:5] == [
+    assert types[:6] == [
         TimingMiddleware,
         TailPreservingResponseLimitingMiddleware,
         ErrorHandlingMiddleware,
         AuditMiddleware,
+        ReadonlyRetryMiddleware,
         SafetyMiddleware,
     ]