From b2f54010ba66373e2ef7a97a8365e744678ad4d6 Mon Sep 17 00:00:00 2001
From: rockymadden <git@rockymadden.com>
Date: Tue, 4 Aug 2026 07:14:10 -0600
Subject: [PATCH 01/10] fix(command-executed): match patterns against
 shell-normalized commands
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`command_pattern` regexes were run only against the raw `bash -lc "..."`
wrapper string, so whichever way the agent happened to quote an argument
(bare, "double", 'single', \"escaped\") leaked into the pattern. Authors
hand-model that escaping and get it subtly wrong: ~173/1557 criteria encode
quote-tolerance, in four inconsistent idioms. skill-flow-paginated-reference-lookup
used `\\?"?` (double-quote only); the agent single-quoted the resource arg,
so a correct multi-call pagination loop scored 0.0 on a gating criterion —
a false negative — while the sibling `nextPage=` criterion matched the same calls.

Normalize before matching: unwrap the `bash`/`sh -c` wrapper and resolve shell
quoting via shlex, then match against BOTH the raw text and the normalized
form (either hit counts). This is additive — nothing that matched before can
stop matching — and lands in the shared `_matching_commands` helper so the
score and the live early-stop trigger stay consistent. Shell operators
(`&&`, `|`, `>`) survive as tokens; unparseable input (unbalanced quotes,
heredocs) falls back to the raw text. Fixes the whole class with no task-YAML
changes.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/criteria/command_executed.py |  82 +++++++++++-
 tests/test_command_executed.py              | 141 ++++++++++++++++++++
 2 files changed, 217 insertions(+), 6 deletions(-)

diff --git a/src/coder_eval/criteria/command_executed.py b/src/coder_eval/criteria/command_executed.py
index e5343ef2..a87da535 100644
--- a/src/coder_eval/criteria/command_executed.py
+++ b/src/coder_eval/criteria/command_executed.py
@@ -3,6 +3,7 @@
 import json
 import logging
 import re
+import shlex
 from typing import TYPE_CHECKING
 
 from coder_eval.criteria.base import BaseCriterion, CheckContext, LiveVerdict, register_criterion
@@ -19,6 +20,73 @@
 # Limit regex search input length to mitigate ReDoS on large command strings
 _MAX_PATTERN_SEARCH_LEN = 2000
 
+# Skip shell-normalization above this size — shlex is linear so this is only a
+# worst-case guard; real telemetry commands are far smaller.
+_MAX_NORMALIZE_LEN = 10 * _MAX_PATTERN_SEARCH_LEN
+
+# Shells whose `-c`/`-lc` payload is the real command we want to match against.
+_SHELL_WRAPPERS = {"bash", "sh"}
+_SHELL_CMD_FLAGS = {"-c", "-lc", "-lic"}
+
+
+def _normalize_shell(cmd_text: str) -> str | None:
+    """Quote-resolved, wrapper-stripped form of a shell command, or None.
+
+    ``command_pattern`` regexes are written against the *logical* command
+    (``uip is resources run list <key> <resource>``), but telemetry records the
+    raw ``bash -lc "..."`` wrapper — so whichever way the agent happened to quote
+    an argument (bare, ``"double"``, ``'single'``, ``\\"escaped\\"``) leaks into
+    the pattern. Authors then hand-model that escaping and get it subtly wrong
+    (e.g. allowing ``"`` but not ``'``), silently under-counting correct calls.
+
+    This unwraps a ``bash``/``sh -c`` wrapper and resolves shell quoting with
+    ``shlex`` so a pattern can match argv semantics regardless of quoting.
+    Shell operators (``&&``, ``|``, ``>``) survive as their own tokens, so
+    patterns that reference them keep working. Returns ``None`` when the text
+    can't be parsed (unbalanced quotes, heredocs); the caller keeps the raw
+    text as a haystack, so nothing that matched before can stop matching.
+    """
+    try:
+        tokens = shlex.split(cmd_text, posix=True)
+    except ValueError:
+        return None
+    if not tokens:
+        return None
+    # Unwrap `bash -lc "<script>"` / `sh -c "<script>"`: the real command is the
+    # argument immediately after the -c/-lc flag.
+    if tokens[0].rsplit("/", 1)[-1] in _SHELL_WRAPPERS:
+        for i in range(1, len(tokens) - 1):
+            tok = tokens[i]
+            if tok in _SHELL_CMD_FLAGS:
+                try:
+                    inner = shlex.split(tokens[i + 1], posix=True)
+                except ValueError:
+                    return None
+                if inner:
+                    tokens = inner
+                break
+            if not tok.startswith("-"):
+                break  # first positional before any -c: not a command wrapper
+    return " ".join(tokens)
+
+
+def _match_haystacks(cmd: "CommandTelemetry", cmd_text: str) -> list[str]:
+    """Strings a pattern may match against for one command.
+
+    Always the raw ``cmd_text``; for Bash, additionally the quote-resolved
+    normalized form (see :func:`_normalize_shell`). Both are length-capped to
+    preserve the ReDoS bound. Matching is "either" — a pattern hits the command
+    if it matches ANY haystack — so normalization only ever repairs a missed
+    match, never removes one. Non-Bash tools serialize params to JSON, where
+    shell tokenization is meaningless, so they are never normalized.
+    """
+    haystacks = [cmd_text[:_MAX_PATTERN_SEARCH_LEN]]
+    if cmd.tool_name == "Bash" and 0 < len(cmd_text) <= _MAX_NORMALIZE_LEN:
+        normalized = _normalize_shell(cmd_text)
+        if normalized is not None and normalized != cmd_text:
+            haystacks.append(normalized[:_MAX_PATTERN_SEARCH_LEN])
+    return haystacks
+
 
 @register_criterion
 class CommandExecutedChecker(BaseCriterion[CommandExecutedCriterion]):
@@ -60,16 +128,18 @@ def _matching_commands(
             else:
                 cmd_text = json.dumps(cmd.parameters)
 
-            # Truncate to mitigate ReDoS on large command strings
-            if len(cmd_text) > _MAX_PATTERN_SEARCH_LEN:
-                cmd_text = cmd_text[:_MAX_PATTERN_SEARCH_LEN]
+            # Match the pattern against the raw command AND its quote-resolved
+            # form, so authors need not encode shell quoting/escaping (which they
+            # do inconsistently, silently under-counting correctly-quoted calls).
+            haystacks = _match_haystacks(cmd, cmd_text)
 
             # Filter by command pattern
-            if pattern is not None and not pattern.search(cmd_text):
+            if pattern is not None and not any(pattern.search(h) for h in haystacks):
                 continue
 
-            # Apply exclusion pattern (skip commands matching the exclusion)
-            if exclude_re is not None and exclude_re.search(cmd_text):
+            # Apply exclusion pattern (skip commands matching the exclusion).
+            # Same both-haystacks logic so exclusion can't be dodged by quoting.
+            if exclude_re is not None and any(exclude_re.search(h) for h in haystacks):
                 continue
 
             # Build a display label for the matched command
diff --git a/tests/test_command_executed.py b/tests/test_command_executed.py
index a936b029..b452624f 100644
--- a/tests/test_command_executed.py
+++ b/tests/test_command_executed.py
@@ -2,6 +2,7 @@
 
 from datetime import datetime
 
+from coder_eval.criteria.command_executed import _normalize_shell
 from coder_eval.evaluation.checker import SuccessChecker
 from coder_eval.models import CommandExecutedCriterion
 from coder_eval.models.results import TurnRecord
@@ -750,3 +751,143 @@ def test_invalid_range_rejected_at_model_level(self):
                 min_count=5,
                 max_count=2,
             )
+
+
+class TestNormalizeShell:
+    """Unit tests for the shell-normalization helper."""
+
+    def test_unwraps_bash_lc_and_resolves_single_quotes(self):
+        raw = (
+            '/bin/bash -lc "uip is resources run list uipath-salesforce-slack '
+            "'curated_channels?types=public_channel,private_channel' --output json\""
+        )
+        assert _normalize_shell(raw) == (
+            "uip is resources run list uipath-salesforce-slack "
+            "curated_channels?types=public_channel,private_channel --output json"
+        )
+
+    def test_unwraps_escaped_double_quotes(self):
+        raw = '/bin/bash -lc "uip is resources run list \\"slack\\" \\"curated_channels\\""'
+        assert _normalize_shell(raw) == "uip is resources run list slack curated_channels"
+
+    def test_bare_command_is_just_requoted(self):
+        assert _normalize_shell("uip is resources run list slack 'curated_channels'") == (
+            "uip is resources run list slack curated_channels"
+        )
+
+    def test_shell_operators_survive_as_tokens(self):
+        raw = "uip maestro flow validate X.flow --output json && uip maestro flow format X.flow"
+        assert _normalize_shell(raw) == raw  # already unquoted; operators kept verbatim
+
+    def test_unbalanced_quotes_return_none(self):
+        assert _normalize_shell("echo 'unterminated") is None
+
+
+class TestShellQuotingNormalization:
+    """Patterns match regardless of how the agent quoted the command.
+
+    Regression for ``skill-flow-paginated-reference-lookup``: the agent
+    paginated ``uip is resources run list <slack> 'curated_channels?...'``
+    correctly (a sibling ``nextPage=`` criterion matched the same calls), but the
+    gating pagination criterion's pattern allowed only a bare or ``\\"``-escaped
+    token — the agent single-quoted the resource arg — so it scored 0.0 (false
+    negative). Normalizing the command before matching fixes the whole class
+    without touching any task YAML.
+    """
+
+    # The exact recorded shape: `bash -lc "..."` wrapper, resource arg in SINGLE
+    # quotes. Second call adds a nextPage token (still single-quoted).
+    _PAGE1 = (
+        '/bin/bash -lc "uip is resources run list uipath-salesforce-slack '
+        "'curated_channels?types=public_channel,private_channel' --connection-id abc --output json\""
+    )
+    _PAGE2 = (
+        '/bin/bash -lc "uip is resources run list uipath-salesforce-slack '
+        "'curated_channels?types=public_channel,private_channel' "
+        "--query 'nextPage=eyJwYWdlIjoyfQ' --output json\""
+    )
+    # The ORIGINAL, unchanged pattern from the task YAML: allows an optional
+    # backslash + optional DOUBLE quote, but no single quote.
+    _YAML_PATTERN = r'uip\s+is\s+resources\s+run\s+list\s+\\?"?uipath-salesforce-slack\\?"?\s+\\?"?curated_channels'
+
+    def test_single_quoted_calls_now_counted_with_original_pattern(self):
+        """The unchanged YAML pattern now counts both single-quoted calls."""
+        sandbox = MockSandbox()
+        turn_records = [
+            _make_turn(
+                [
+                    _make_command(tool_name="Bash", parameters={"command": self._PAGE1}, tool_id="t1"),
+                    _make_command(tool_name="Bash", parameters={"command": self._PAGE2}, tool_id="t2"),
+                ]
+            )
+        ]
+        criterion = CommandExecutedCriterion(
+            description="paginated curated_channels list ran >1x",
+            tool_name="Bash",
+            command_pattern=self._YAML_PATTERN,
+            min_count=2,
+        )
+        result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
+        assert result.score == 1.0
+        assert "2/2" in result.details
+
+    def test_escaped_double_quote_form_still_matches(self):
+        """Backward compat: the escaping style the pattern anticipated still hits."""
+        sandbox = MockSandbox()
+        raw = (
+            '/bin/bash -lc "uip is resources run list \\"uipath-salesforce-slack\\" '
+            '\\"curated_channels?types=x\\" --output json"'
+        )
+        turn_records = [_make_turn([_make_command(tool_name="Bash", parameters={"command": raw})])]
+        criterion = CommandExecutedCriterion(
+            description="curated_channels list ran",
+            tool_name="Bash",
+            command_pattern=self._YAML_PATTERN,
+            min_count=1,
+        )
+        result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
+        assert result.score == 1.0
+
+    def test_shell_operator_pattern_still_matches(self):
+        """`&&`/`|` patterns keep working — operators survive normalization."""
+        sandbox = MockSandbox()
+        cmd = "/bin/bash -lc 'uip maestro flow validate X.flow --output json && uip maestro flow format X.flow'"
+        turn_records = [_make_turn([_make_command(tool_name="Bash", parameters={"command": cmd})])]
+        criterion = CommandExecutedCriterion(
+            description="validate then format",
+            tool_name="Bash",
+            command_pattern=r"uip\s+maestro\s+flow\s+validate.*&&.*uip\s+maestro\s+flow\s+format",
+            min_count=1,
+        )
+        result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
+        assert result.score == 1.0
+
+    def test_unbalanced_quotes_fall_back_to_raw_without_crashing(self):
+        """A command shlex can't parse still matches against its raw text."""
+        sandbox = MockSandbox()
+        turn_records = [_make_turn([_make_command(tool_name="Bash", parameters={"command": "echo 'unterminated"})])]
+        criterion = CommandExecutedCriterion(
+            description="echo ran",
+            tool_name="Bash",
+            command_pattern=r"echo\s+",
+            min_count=1,
+        )
+        result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
+        assert result.score == 1.0
+
+    def test_negative_assertion_not_dodged_by_quoting(self):
+        """A quote-obfuscated retired call is still caught by a max_count=0 gate."""
+        sandbox = MockSandbox()
+        # `'uip' or users list` doesn't match `uip\s+or` on the raw text (a quote
+        # sits right after uip), but the normalized form `uip or users list` does.
+        obfuscated = "/bin/bash -lc \"'uip' or users list\""
+        turn_records = [_make_turn([_make_command(tool_name="Bash", parameters={"command": obfuscated})])]
+        criterion = CommandExecutedCriterion(
+            description="must NOT use retired `uip or users list`",
+            tool_name="Bash",
+            command_pattern=r"uip\s+or\s+users\s+list",
+            min_count=0,
+            max_count=0,
+        )
+        result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
+        assert result.score == 0.0

From d26fdad91544a0f7415ba00d1ddf11bad3b8582d Mon Sep 17 00:00:00 2001
From: rockymadden <git@rockymadden.com>
Date: Wed, 5 Aug 2026 08:14:48 -0600
Subject: [PATCH 02/10] fix(command-executed): narrow command param to str
 before shell-normalizing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`CommandTelemetry.parameters` is `dict[str, Any]`, and a `command` value is
not guaranteed to be a `str`: Codex sub-agent rollout recovery can carry it as
an argv *list*. That list reached `shlex.split`/slicing and raised
`AttributeError: 'list' object has no attribute 'read'`, which aborted
`_matching_commands` for the entire trajectory and zeroed an otherwise-passing
`command_executed` criterion.

Narrow the extraction (and the display-label) site with `isinstance(raw, str)`
— mirroring criteria/skill_triggered.py — so a non-str `command` falls back to
the JSON-serialized params blob instead of crashing.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/criteria/command_executed.py | 18 ++++++---
 tests/test_command_executed.py              | 44 +++++++++++++++++++++
 2 files changed, 56 insertions(+), 6 deletions(-)

diff --git a/src/coder_eval/criteria/command_executed.py b/src/coder_eval/criteria/command_executed.py
index a87da535..5679cedb 100644
--- a/src/coder_eval/criteria/command_executed.py
+++ b/src/coder_eval/criteria/command_executed.py
@@ -122,9 +122,15 @@ def _matching_commands(
             if criterion.require_success and cmd.result_status != "success":
                 continue
 
-            # Extract text for pattern matching (Bash: command param; others: JSON-serialized params)
-            if cmd.tool_name == "Bash" and cmd.parameters.get("command"):
-                cmd_text = cmd.parameters["command"]
+            # Extract text for pattern matching (Bash: command param; others: JSON-serialized params).
+            # ``parameters`` is ``dict[str, Any]``, and a ``command`` value is not
+            # guaranteed to be a ``str`` — Codex sub-agent rollout recovery can carry
+            # it as an argv *list* (codex_agent.py). Narrow with ``isinstance`` so a
+            # non-``str`` value never reaches ``shlex.split``/slicing (which would raise
+            # ``AttributeError`` and zero the whole criterion); fall back to the JSON blob.
+            raw_command = cmd.parameters.get("command")
+            if cmd.tool_name == "Bash" and isinstance(raw_command, str) and raw_command:
+                cmd_text = raw_command
             else:
                 cmd_text = json.dumps(cmd.parameters)
 
@@ -142,9 +148,9 @@ def _matching_commands(
             if exclude_re is not None and any(exclude_re.search(h) for h in haystacks):
                 continue
 
-            # Build a display label for the matched command
-            if cmd.tool_name == "Bash" and cmd.parameters.get("command"):
-                label = cmd.parameters["command"]
+            # Build a display label for the matched command (same narrowing as above)
+            if cmd.tool_name == "Bash" and isinstance(raw_command, str) and raw_command:
+                label = raw_command
             else:
                 label = f"{cmd.tool_name}({json.dumps(cmd.parameters)[:80]})"
             matching.append(label)
diff --git a/tests/test_command_executed.py b/tests/test_command_executed.py
index b452624f..69b7ab8b 100644
--- a/tests/test_command_executed.py
+++ b/tests/test_command_executed.py
@@ -514,6 +514,50 @@ def test_non_bash_tool(self):
 
         assert result.score == 1.0
 
+    def test_non_str_command_does_not_crash_criterion(self):
+        """A non-``str`` ``command`` (Codex argv array) must not zero the criterion.
+
+        Codex sub-agent rollout recovery can carry ``command`` as an argv *list*
+        (codex_agent.py), which reaches ``CommandTelemetry.parameters`` verbatim.
+        Before the ``isinstance`` narrow, the list fell through to
+        ``shlex.split(list)`` -> ``AttributeError: 'list' object has no attribute
+        'read'``, which aborted ``_matching_commands`` for the entire trajectory
+        and scored a pattern-less, otherwise-passing criterion 0.0.
+        """
+        sandbox = MockSandbox()
+        turn_records = [
+            _make_turn(
+                [
+                    _make_command(tool_name="Bash", parameters={"command": ["bash", "-lc", "ls"]}, tool_id="t1"),
+                    _make_command(tool_name="Bash", parameters={"command": "echo hi"}, tool_id="t2"),
+                ]
+            )
+        ]
+        # Pattern-less: both Bash commands count; the argv-list one must not crash.
+        criterion = CommandExecutedCriterion(description="ran bash", tool_name="Bash", min_count=1)
+        result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
+        assert result.error is None
+        assert result.score == 1.0
+        assert "2/1" in result.details
+
+    def test_non_str_command_still_matches_sibling_by_pattern(self):
+        """One argv-list ``command`` must not poison a pattern match on its sibling."""
+        sandbox = MockSandbox()
+        turn_records = [
+            _make_turn(
+                [
+                    _make_command(tool_name="Bash", parameters={"command": ["bash", "-lc", "ls"]}, tool_id="t1"),
+                    _make_command(tool_name="Bash", parameters={"command": "uip run foo"}, tool_id="t2"),
+                ]
+            )
+        ]
+        criterion = CommandExecutedCriterion(
+            description="ran uip", tool_name="Bash", command_pattern=r"uip\s+run", min_count=1
+        )
+        result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
+        assert result.error is None
+        assert result.score == 1.0
+
     def test_crashed_turn_commands_are_counted(self):
         """Commands from crashed partial turns count toward min_count.
 

From 26c406241354b9c3a83ccc401cfd07747c3ad5b7 Mon Sep 17 00:00:00 2001
From: rockymadden <git@rockymadden.com>
Date: Wed, 5 Aug 2026 08:17:21 -0600
Subject: [PATCH 03/10] fix(command-executed): keep whole argv-joined payload
 in shell unwrap
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The `bash -lc` unwrap kept only `tokens[i+1]`, so an argv-joined (unquoted)
payload collapsed to its first word: Codex sub-agent rollout recovery joins
argv WITHOUT re-quoting (codex_agent.py), so `bash -lc uip is resources ...`
normalized to just `uip` — making the shell-normalization fix a silent no-op
on that path, and letting the degenerate one-word haystack newly (and falsely)
satisfy an anchored pattern like `^uip$`.

Rejoin everything after the -c/-lc flag: re-split the single quoted-script
token to resolve inner quotes, but keep every token of the already-split
argv-joined form. Also strips the wrapper cleanly for an empty script
(`bash -lc ""` -> `''` instead of the odd `bash -lc `).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/criteria/command_executed.py | 25 ++++++----
 tests/test_command_executed.py              | 53 +++++++++++++++++++++
 2 files changed, 70 insertions(+), 8 deletions(-)

diff --git a/src/coder_eval/criteria/command_executed.py b/src/coder_eval/criteria/command_executed.py
index 5679cedb..83575d52 100644
--- a/src/coder_eval/criteria/command_executed.py
+++ b/src/coder_eval/criteria/command_executed.py
@@ -52,18 +52,27 @@ def _normalize_shell(cmd_text: str) -> str | None:
         return None
     if not tokens:
         return None
-    # Unwrap `bash -lc "<script>"` / `sh -c "<script>"`: the real command is the
-    # argument immediately after the -c/-lc flag.
+    # Unwrap `bash -lc "<script>"` / `sh -c "<script>"`: the real command is
+    # everything after the -c/-lc flag.
     if tokens[0].rsplit("/", 1)[-1] in _SHELL_WRAPPERS:
         for i in range(1, len(tokens) - 1):
             tok = tokens[i]
             if tok in _SHELL_CMD_FLAGS:
-                try:
-                    inner = shlex.split(tokens[i + 1], posix=True)
-                except ValueError:
-                    return None
-                if inner:
-                    tokens = inner
+                rest = tokens[i + 1 :]
+                if len(rest) == 1:
+                    # Quoted whole-script form (`bash -lc "uip ... 'arg' ..."`):
+                    # the script is a single token that may still hold inner
+                    # quotes — re-split to resolve them.
+                    try:
+                        tokens = shlex.split(rest[0], posix=True)
+                    except ValueError:
+                        return None
+                else:
+                    # Argv-joined form: Codex rollout recovery joins argv WITHOUT
+                    # re-quoting (codex_agent.py), so `bash -lc uip is resources ...`
+                    # already arrives split — keep every token instead of
+                    # collapsing to the first word.
+                    tokens = rest
                 break
             if not tok.startswith("-"):
                 break  # first positional before any -c: not a command wrapper
diff --git a/tests/test_command_executed.py b/tests/test_command_executed.py
index 69b7ab8b..1674953d 100644
--- a/tests/test_command_executed.py
+++ b/tests/test_command_executed.py
@@ -826,6 +826,19 @@ def test_shell_operators_survive_as_tokens(self):
     def test_unbalanced_quotes_return_none(self):
         assert _normalize_shell("echo 'unterminated") is None
 
+    def test_argv_joined_payload_is_not_collapsed_to_first_word(self):
+        """Codex rollout recovery joins argv WITHOUT re-quoting (codex_agent.py).
+
+        The wrapper unwrap must keep every token after ``-lc``, not just the
+        first — otherwise ``bash -lc uip is resources ...`` collapses to ``uip``,
+        making the fix a silent no-op on the sub-agent path.
+        """
+        raw = "bash -lc uip is resources run list slack curated_channels --output json"
+        assert _normalize_shell(raw) == "uip is resources run list slack curated_channels --output json"
+
+    def test_argv_joined_short_command(self):
+        assert _normalize_shell("bash -c echo hi there") == "echo hi there"
+
 
 class TestShellQuotingNormalization:
     """Patterns match regardless of how the agent quoted the command.
@@ -919,6 +932,46 @@ def test_unbalanced_quotes_fall_back_to_raw_without_crashing(self):
         result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
         assert result.score == 1.0
 
+    def test_argv_joined_command_matches_full_pattern(self):
+        """The argv-joined (unquoted) shape matches a whole-command pattern.
+
+        This is Codex's sub-agent rollout-recovery telemetry — argv joined with
+        spaces and no re-quoting. Before the unwrap fix it collapsed to the first
+        word, so this pattern scored 0.0.
+        """
+        sandbox = MockSandbox()
+        cmd = "bash -lc uip is resources run list uipath-salesforce-slack curated_channels --output json"
+        turn_records = [_make_turn([_make_command(tool_name="Bash", parameters={"command": cmd})])]
+        criterion = CommandExecutedCriterion(
+            description="listed curated_channels",
+            tool_name="Bash",
+            command_pattern=r"uip\s+is\s+resources\s+run\s+list\s+\S+\s+curated_channels",
+            min_count=1,
+        )
+        result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
+        assert result.score == 1.0
+
+    def test_argv_joined_not_collapsed_into_false_anchored_match(self):
+        """The collapse-to-first-word bug could make ``^git$`` match ``git push ...``.
+
+        The real command is ``git push --force origin main``; an anchored
+        ``^git$`` negative assertion must PASS (score 1.0). The old code collapsed
+        the payload to the single token ``git``, which matched ``^git$`` and
+        force-failed the gate for a command the agent never actually ran bare.
+        """
+        sandbox = MockSandbox()
+        cmd = "bash -lc git push --force origin main"
+        turn_records = [_make_turn([_make_command(tool_name="Bash", parameters={"command": cmd})])]
+        criterion = CommandExecutedCriterion(
+            description="must NOT run bare `git`",
+            tool_name="Bash",
+            command_pattern=r"^git$",
+            min_count=0,
+            max_count=0,
+        )
+        result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
+        assert result.score == 1.0
+
     def test_negative_assertion_not_dodged_by_quoting(self):
         """A quote-obfuscated retired call is still caught by a max_count=0 gate."""
         sandbox = MockSandbox()

From 4f578014f528ae7b970558cb19e94050c27cbb86 Mon Sep 17 00:00:00 2001
From: rockymadden <git@rockymadden.com>
Date: Wed, 5 Aug 2026 08:23:50 -0600
Subject: [PATCH 04/10] fix(command-executed): recognize shell wrappers by
 predicate, not allowlist

The enumerated `_SHELL_WRAPPERS = {"bash","sh"}` / `_SHELL_CMD_FLAGS =
{"-c","-lc","-lic"}` sets omitted `zsh` (Codex's login shell on macOS, see
codex_agent.py) and the common `-ic` while listing the exotic `-lic`. On those
hosts the shell-normalization silently reverted to the pre-fix false-negative
behaviour, so identical agent behaviour scored differently by host shell.

Replace both enumerated sets with predicates over the open shell/flag domain:
`_is_shell_program` matches any basename ending in `sh` (bash/sh/zsh/dash/ksh);
`_is_command_flag` matches a short-option cluster whose letters are alphabetic
and include `c` (`-c`/`-lc`/`-ic`/`-lic`, and the split `bash -l -c` form),
while rejecting `--long` options and `git -c <config>`. Matching stays additive
(raw text remains a haystack), so favouring recall is safe. Adds one test per
wrapper form so the domain cannot silently rot.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/criteria/command_executed.py | 31 +++++++++++++++---
 tests/test_command_executed.py              | 35 +++++++++++++++++++++
 2 files changed, 61 insertions(+), 5 deletions(-)

diff --git a/src/coder_eval/criteria/command_executed.py b/src/coder_eval/criteria/command_executed.py
index 83575d52..50f0db71 100644
--- a/src/coder_eval/criteria/command_executed.py
+++ b/src/coder_eval/criteria/command_executed.py
@@ -24,9 +24,30 @@
 # worst-case guard; real telemetry commands are far smaller.
 _MAX_NORMALIZE_LEN = 10 * _MAX_PATTERN_SEARCH_LEN
 
-# Shells whose `-c`/`-lc` payload is the real command we want to match against.
-_SHELL_WRAPPERS = {"bash", "sh"}
-_SHELL_CMD_FLAGS = {"-c", "-lc", "-lic"}
+
+def _is_shell_program(arg0: str) -> bool:
+    """True if argv[0]'s basename looks like a POSIX shell.
+
+    A predicate rather than an enumerated allowlist: the set of shells is open
+    (``bash``/``sh`` on Linux, ``zsh`` on macOS — Codex shells through the host's
+    login shell, see codex_agent.py — plus ``dash``/``ksh``/…), and every common
+    shell basename ends in ``sh``. Matching is additive (the raw text stays a
+    haystack), so favouring recall over a hand-maintained list is safe.
+    """
+    return arg0.rsplit("/", 1)[-1].endswith("sh")
+
+
+def _is_command_flag(tok: str) -> bool:
+    """True for a short-option cluster carrying a ``-c`` command string.
+
+    Covers ``-c``, ``-lc``, ``-ic``, ``-lic`` (login/interactive + command) in
+    any order — a single ``-``-prefixed token whose option letters are all
+    alphabetic and include ``c``. ``--long`` options and ``-o=val`` forms are
+    rejected, so this is the first flag that actually introduces the command
+    string. The combined and split (``bash -l -c``) forms both work: a non-``c``
+    short flag like ``-l`` simply isn't the command flag and the scan continues.
+    """
+    return len(tok) >= 2 and tok[0] == "-" and tok[1] != "-" and tok[1:].isalpha() and "c" in tok[1:]
 
 
 def _normalize_shell(cmd_text: str) -> str | None:
@@ -54,10 +75,10 @@ def _normalize_shell(cmd_text: str) -> str | None:
         return None
     # Unwrap `bash -lc "<script>"` / `sh -c "<script>"`: the real command is
     # everything after the -c/-lc flag.
-    if tokens[0].rsplit("/", 1)[-1] in _SHELL_WRAPPERS:
+    if _is_shell_program(tokens[0]):
         for i in range(1, len(tokens) - 1):
             tok = tokens[i]
-            if tok in _SHELL_CMD_FLAGS:
+            if _is_command_flag(tok):
                 rest = tokens[i + 1 :]
                 if len(rest) == 1:
                     # Quoted whole-script form (`bash -lc "uip ... 'arg' ..."`):
diff --git a/tests/test_command_executed.py b/tests/test_command_executed.py
index 1674953d..c4cc6764 100644
--- a/tests/test_command_executed.py
+++ b/tests/test_command_executed.py
@@ -839,6 +839,41 @@ def test_argv_joined_payload_is_not_collapsed_to_first_word(self):
     def test_argv_joined_short_command(self):
         assert _normalize_shell("bash -c echo hi there") == "echo hi there"
 
+    def test_every_wrapper_form_is_unwrapped(self):
+        """One case per shell/flag shape the agents emit — the allowlist can't rot.
+
+        The predicate replaced an enumerated allowlist that omitted ``zsh``
+        (Codex's shell on macOS, codex_agent.py) and ``-ic`` while listing the
+        exotic ``-lic``; on those hosts the normalization silently reverted to
+        the pre-fix false-negative behaviour. Each entry must strip the wrapper.
+        """
+        cases = {
+            # zsh — Codex's default login shell on macOS
+            '/bin/zsh -lc "uip is resources run list slack curated_channels"': (
+                "uip is resources run list slack curated_channels"
+            ),
+            'zsh -lc "echo hi"': "echo hi",
+            'sh -c "echo hi"': "echo hi",
+            'dash -c "echo hi"': "echo hi",
+            'ksh -c "echo hi"': "echo hi",
+            'bash -ic "echo hi"': "echo hi",  # interactive + command
+            '/usr/bin/bash -lic "echo hi"': "echo hi",  # login + interactive + command
+            "bash -l -c 'echo hi'": "echo hi",  # split login/command flags
+        }
+        for raw, expected in cases.items():
+            assert _normalize_shell(raw) == expected, raw
+
+    def test_non_shell_arg0_is_not_unwrapped(self):
+        """A non-shell program (basename not ending in ``sh``) is only re-quoted.
+
+        ``git -c <config>`` is the motivating case: ``-c`` is a real git flag, but
+        because ``git`` is not a shell the payload must NOT be unwrapped.
+        """
+        assert _normalize_shell("git -c user.name=x status") == "git -c user.name=x status"
+        assert _normalize_shell("uip is resources run list slack 'curated_channels'") == (
+            "uip is resources run list slack curated_channels"
+        )
+
 
 class TestShellQuotingNormalization:
     """Patterns match regardless of how the agent quoted the command.

From 7d42c5fa204e6242e2dedab5c17854b359c3df28 Mon Sep 17 00:00:00 2001
From: rockymadden <git@rockymadden.com>
Date: Wed, 5 Aug 2026 08:31:12 -0600
Subject: [PATCH 05/10] refactor(command-executed): total _match_haystacks,
 shared window, memoized
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Make _match_haystacks total and stop it re-deriving caller state:

- Signature is now `(cmd_text, *, is_shell)`; `is_shell` is decided once at the
  single extraction site (a Bash tool whose `command` is a non-empty `str`).
  Previously the helper re-derived shell-ness from `tool_name == "Bash"` alone,
  so a Bash record with a missing/empty `command` had its JSON params blob
  shlex-tokenized — the stripped JSON quotes could newly satisfy an
  `exclude_pattern` and drop the command below `min_count` (flip 1.0 -> 0.0).
- Normalize the already-truncated `cmd_text[:_MAX_PATTERN_SEARCH_LEN]` so both
  haystacks describe the same 2000-char window; quote-stripping can no longer
  slide content from past the cap into the match. This also bounds `shlex` input
  for free, retiring the `_MAX_NORMALIZE_LEN` guard (whose value was coupled to
  an unrelated ReDoS constant and whose off-by-one silently changed semantics at
  the cap).
- Memoize `_normalize_shell` (pure function): `EarlyStopWatcher._evaluate`
  re-scans the whole accumulated trajectory on every tool-call event, so the
  same command was lexed O(n^2) times per run — the cache collapses that to
  once per distinct command string.

Tests: cover every previously-uncovered `_normalize_shell` branch (empty/
whitespace input, inner unbalanced-quote fallback, non-wrapper positional,
flags-without-command), the is_shell divergence (Bash-without-command not
excluded), the shared-window guard, and memoization (via cache_info, not
timing). Module is now at 100% statement + branch coverage.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 src/coder_eval/criteria/command_executed.py | 53 ++++++++-----
 tests/test_command_executed.py              | 83 ++++++++++++++++++++-
 2 files changed, 117 insertions(+), 19 deletions(-)

diff --git a/src/coder_eval/criteria/command_executed.py b/src/coder_eval/criteria/command_executed.py
index 50f0db71..2cb0e746 100644
--- a/src/coder_eval/criteria/command_executed.py
+++ b/src/coder_eval/criteria/command_executed.py
@@ -4,6 +4,7 @@
 import logging
 import re
 import shlex
+from functools import lru_cache
 from typing import TYPE_CHECKING
 
 from coder_eval.criteria.base import BaseCriterion, CheckContext, LiveVerdict, register_criterion
@@ -17,13 +18,11 @@
 
 logger = logging.getLogger(__name__)
 
-# Limit regex search input length to mitigate ReDoS on large command strings
+# Limit regex search input length to mitigate ReDoS on large command strings.
+# Normalization runs over this same truncated window (see _match_haystacks), so
+# shlex never sees more than this many chars and needs no separate size guard.
 _MAX_PATTERN_SEARCH_LEN = 2000
 
-# Skip shell-normalization above this size — shlex is linear so this is only a
-# worst-case guard; real telemetry commands are far smaller.
-_MAX_NORMALIZE_LEN = 10 * _MAX_PATTERN_SEARCH_LEN
-
 
 def _is_shell_program(arg0: str) -> bool:
     """True if argv[0]'s basename looks like a POSIX shell.
@@ -50,6 +49,7 @@ def _is_command_flag(tok: str) -> bool:
     return len(tok) >= 2 and tok[0] == "-" and tok[1] != "-" and tok[1:].isalpha() and "c" in tok[1:]
 
 
+@lru_cache(maxsize=1024)
 def _normalize_shell(cmd_text: str) -> str | None:
     """Quote-resolved, wrapper-stripped form of a shell command, or None.
 
@@ -66,6 +66,11 @@ def _normalize_shell(cmd_text: str) -> str | None:
     patterns that reference them keep working. Returns ``None`` when the text
     can't be parsed (unbalanced quotes, heredocs); the caller keeps the raw
     text as a haystack, so nothing that matched before can stop matching.
+
+    Memoized (pure function of ``cmd_text``): the early-stop watcher re-scans the
+    whole accumulated trajectory on every tool-call event, so the same command is
+    normalized many times per run — the cache collapses that to once per distinct
+    (already-truncated) command string.
     """
     try:
         tokens = shlex.split(cmd_text, posix=True)
@@ -100,21 +105,29 @@ def _normalize_shell(cmd_text: str) -> str | None:
     return " ".join(tokens)
 
 
-def _match_haystacks(cmd: "CommandTelemetry", cmd_text: str) -> list[str]:
+def _match_haystacks(cmd_text: str, *, is_shell: bool) -> list[str]:
     """Strings a pattern may match against for one command.
 
-    Always the raw ``cmd_text``; for Bash, additionally the quote-resolved
-    normalized form (see :func:`_normalize_shell`). Both are length-capped to
-    preserve the ReDoS bound. Matching is "either" — a pattern hits the command
-    if it matches ANY haystack — so normalization only ever repairs a missed
-    match, never removes one. Non-Bash tools serialize params to JSON, where
-    shell tokenization is meaningless, so they are never normalized.
+    Always the raw ``cmd_text`` truncated to the ReDoS bound; when ``is_shell``,
+    additionally the quote-resolved, wrapper-stripped form of that **same
+    truncated window** (see :func:`_normalize_shell`). Normalizing the already-
+    truncated slice keeps both haystacks describing the same window, so quote-
+    stripping can never slide content from past the cap into the match, and
+    caps ``shlex`` input at ``_MAX_PATTERN_SEARCH_LEN`` for free. Matching is
+    "either" — a pattern hits the command if it matches ANY haystack.
+
+    ``is_shell`` is decided once by the caller (a Bash tool whose ``command`` is
+    a non-empty ``str``) and passed in, rather than re-derived here from
+    ``tool_name`` alone: a Bash record with a missing/empty ``command`` serializes
+    its params to JSON, where shell tokenization is meaningless, and must NOT be
+    normalized (else stripped JSON quotes could newly satisfy an exclusion).
     """
-    haystacks = [cmd_text[:_MAX_PATTERN_SEARCH_LEN]]
-    if cmd.tool_name == "Bash" and 0 < len(cmd_text) <= _MAX_NORMALIZE_LEN:
-        normalized = _normalize_shell(cmd_text)
-        if normalized is not None and normalized != cmd_text:
-            haystacks.append(normalized[:_MAX_PATTERN_SEARCH_LEN])
+    window = cmd_text[:_MAX_PATTERN_SEARCH_LEN]
+    haystacks = [window]
+    if is_shell:
+        normalized = _normalize_shell(window)
+        if normalized is not None and normalized != window:
+            haystacks.append(normalized)
     return haystacks
 
 
@@ -161,13 +174,17 @@ def _matching_commands(
             raw_command = cmd.parameters.get("command")
             if cmd.tool_name == "Bash" and isinstance(raw_command, str) and raw_command:
                 cmd_text = raw_command
+                is_shell = True
             else:
                 cmd_text = json.dumps(cmd.parameters)
+                is_shell = False
 
             # Match the pattern against the raw command AND its quote-resolved
             # form, so authors need not encode shell quoting/escaping (which they
             # do inconsistently, silently under-counting correctly-quoted calls).
-            haystacks = _match_haystacks(cmd, cmd_text)
+            # ``is_shell`` (decided once, above) tells the helper whether cmd_text
+            # is a shell command — it must not re-derive that from tool_name alone.
+            haystacks = _match_haystacks(cmd_text, is_shell=is_shell)
 
             # Filter by command pattern
             if pattern is not None and not any(pattern.search(h) for h in haystacks):
diff --git a/tests/test_command_executed.py b/tests/test_command_executed.py
index c4cc6764..9c6a3711 100644
--- a/tests/test_command_executed.py
+++ b/tests/test_command_executed.py
@@ -2,7 +2,7 @@
 
 from datetime import datetime
 
-from coder_eval.criteria.command_executed import _normalize_shell
+from coder_eval.criteria.command_executed import _MAX_PATTERN_SEARCH_LEN, _match_haystacks, _normalize_shell
 from coder_eval.evaluation.checker import SuccessChecker
 from coder_eval.models import CommandExecutedCriterion
 from coder_eval.models.results import TurnRecord
@@ -874,6 +874,45 @@ def test_non_shell_arg0_is_not_unwrapped(self):
             "uip is resources run list slack curated_channels"
         )
 
+    def test_empty_and_whitespace_input_return_none(self):
+        """Empty / whitespace-only input has no tokens -> None (benign, not an error)."""
+        assert _normalize_shell("") is None
+        assert _normalize_shell("   ") is None
+
+    def test_inner_unbalanced_quotes_return_none(self):
+        """A wrapper whose script token can't be re-split falls back to None."""
+        # Outer double quotes balance, so the script token is `echo 'unterminated`;
+        # re-splitting that raises ValueError on the stray single quote.
+        assert _normalize_shell('bash -lc "echo \'unterminated"') is None
+
+    def test_non_wrapper_positional_returns_verbatim(self):
+        """A shell with a positional before any -c is a script invocation, not `-c`.
+
+        `bash script.sh -c foo` runs the file `script.sh`; the later `-c` is an
+        argument to the script, not a command flag, so nothing is unwrapped.
+        """
+        assert _normalize_shell("bash script.sh -c foo") == "bash script.sh -c foo"
+
+    def test_shell_with_flags_but_no_command_flag_is_verbatim(self):
+        """A shell invoked with only non-``-c`` flags (no command) unwraps nothing.
+
+        The wrapper scan exhausts without finding a command flag or a positional,
+        so the tokens are returned as-is.
+        """
+        assert _normalize_shell("bash --norc -i") == "bash --norc -i"
+
+    def test_is_memoized(self):
+        """The hot early-stop path re-scans the trajectory; normalize once per command.
+
+        Counting via ``cache_info`` (not wall-clock) so the guard can't flake.
+        """
+        _normalize_shell.cache_clear()
+        raw = "bash -lc 'echo hello world'"
+        first = _normalize_shell(raw)
+        second = _normalize_shell(raw)
+        assert first == second == "echo hello world"
+        assert _normalize_shell.cache_info().hits >= 1  # second call served from cache
+
 
 class TestShellQuotingNormalization:
     """Patterns match regardless of how the agent quoted the command.
@@ -1007,6 +1046,48 @@ def test_argv_joined_not_collapsed_into_false_anchored_match(self):
         result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
         assert result.score == 1.0
 
+    def test_bash_record_without_command_is_not_shell_normalized(self):
+        """A Bash record with no ``command`` key must not have its JSON params tokenized.
+
+        ``_match_haystacks`` is now told whether cmd_text is a shell command
+        (``is_shell``, decided once at the extraction site) instead of
+        re-deriving it from ``tool_name == "Bash"`` alone. A Bash record whose
+        ``command`` is missing (reachable on the Codex path) falls back to the
+        JSON blob and must NOT be normalized — otherwise shlex strips the JSON
+        quotes, adding a haystack that can newly satisfy an ``exclude_pattern``
+        and wrongly drop the command below ``min_count``.
+        """
+        sandbox = MockSandbox()
+        # Bash tool, but params carry a `description`, not a `command`.
+        turn_records = [
+            _make_turn([_make_command(tool_name="Bash", parameters={"description": "run the pytest suite"})])
+        ]
+        criterion = CommandExecutedCriterion(
+            description="counts the bash record",
+            tool_name="Bash",
+            # Matches the shlex-stripped JSON (`{description: run ...}`) but NOT the
+            # raw JSON (`{"description": "run ...}`); with the is_shell fix the raw
+            # is the only haystack, so the command is not excluded.
+            exclude_pattern=r"description:\s+run",
+            min_count=1,
+        )
+        result = SuccessChecker(sandbox).check(criterion, turn_records=turn_records)
+        assert result.score == 1.0
+
+    def test_normalized_haystack_shares_the_raw_truncation_window(self):
+        """Both haystacks describe the same <=2000-char window (no past-cap leak).
+
+        Previously the raw haystack was truncated at 2000 chars but normalization
+        ran over the FULL command, so quote-stripping could slide content from
+        past the cap into the normalized haystack — a task relying on the
+        2000-char bound changed verdict. Normalization now runs over the
+        already-truncated window.
+        """
+        cmd = "bash -lc " + ("word " * 600) + "TARGET"  # TARGET sits well past 2000 chars
+        haystacks = _match_haystacks(cmd, is_shell=True)
+        assert all(len(h) <= _MAX_PATTERN_SEARCH_LEN for h in haystacks)
+        assert not any("TARGET" in h for h in haystacks)
+
     def test_negative_assertion_not_dodged_by_quoting(self):
         """A quote-obfuscated retired call is still caught by a max_count=0 gate."""
         sandbox = MockSandbox()

From 96d26a40ea5f67c903c841ff13b4e9aa4366c953 Mon Sep 17 00:00:00 2001
From: rockymadden <git@rockymadden.com>
Date: Wed, 5 Aug 2026 08:48:47 -0600
Subject: [PATCH 06/10] docs(command-executed): document shell-normalization
 contract + gate it (CE030)

The shell-normalization change altered what `command_pattern`/`exclude_pattern`
match on unedited task YAML, but the author-facing contract was untouched.

- `command_pattern` / `exclude_pattern` Field descriptions now state that a Bash
  command is matched against the raw text OR its shell-normalized (shlex-resolved,
  wrapper-stripped) form, and that `exclude_pattern` therefore also excludes
  quote-obfuscated variants.
- TASK_DEFINITION_GUIDE gains a "Shell normalization" paragraph under
  `command_executed` spelling out the raw-OR-normalized behavior and, crucially,
  that it is NOT purely additive: a normalized haystack can newly satisfy an
  exclusion or a `max_count: 0` gate, so cross-repo suites that hand-encoded
  quote tolerance should re-baseline. Also fills in the previously YAML-only
  field references (command_executed / reference_comparison field tables,
  run_command `score_from_stdout`, the judges' `capture_transcript` /
  `max_transcript_chars`).
- Corrects the `_normalize_shell` docstring's false invariants (heredocs parse
  fine; matching is not purely additive).
- Extends CE030 doc/schema-parity to the `SuccessCriterion` union members
  (enumerated, no recursion), so the next criterion-semantics change cannot ship
  with an undocumented field. Registry now covers 18 models, 0 undocumented.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 docs/TASK_DEFINITION_GUIDE.md               | 22 +++++++++
 src/coder_eval/criteria/command_executed.py | 18 +++++--
 src/coder_eval/models/criteria.py           | 13 ++++-
 tests/lint/doc_schema_parity.py             | 54 +++++++++++++++++----
 4 files changed, 90 insertions(+), 17 deletions(-)

diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md
index 69cbde15..fb6e1ac4 100644
--- a/docs/TASK_DEFINITION_GUIDE.md
+++ b/docs/TASK_DEFINITION_GUIDE.md
@@ -768,6 +768,7 @@ Runs a command and checks the exit code, with optional stdout matching. **Binary
 | `expected_exit_code` | 0 | Expected exit code |
 | `expected_stdout` | `null` | When set, stdout is also checked |
 | `stdout_match` | `"exact"` | Match mode: `exact` (stripped), `contains` (substring), `regex` (pattern) |
+| `score_from_stdout` | `false` | Read a float score (0.0–1.0) from the first stdout line (remaining lines become details); a non-zero exit code or a parse failure scores 0.0. Mutually exclusive with `expected_stdout`. |
 
 ### `file_matches_regex`
 
@@ -795,6 +796,12 @@ Compares agent's code with a reference solution using similarity scoring. **Cont
   weight: 2.0
 ```
 
+| Field | Default | Description |
+|-------|---------|-------------|
+| `agent_file` | *required* | Path to the agent's generated file (relative to the sandbox root). |
+| `comparison_method` | `"ast"` | `ast` (structure), `token` (text), or `complexity` (metrics). |
+| `similarity_threshold` | 0.8 | Minimum similarity score to pass (0.0–1.0). |
+
 **Comparison methods:**
 - `ast` — Abstract Syntax Tree similarity (structure-based)
 - `token` — Token-based similarity (implementation details)
@@ -813,6 +820,17 @@ Checks whether the agent executed specific tools/commands during evaluation. Ins
   description: "Agent must use curl to fetch weather"
 ```
 
+| Field | Default | Description |
+|-------|---------|-------------|
+| `tool_name` | `null` | Tool-name filter (e.g. `Bash`); `null` counts any tool. |
+| `command_pattern` | `null` | Regex to match the command; `null` matches any command. Matched with shell normalization (see below). |
+| `min_count` | 1 | Minimum matching commands required. `0` permits zero matches — combine with `max_count: 0` to assert a command must **NOT** run. |
+| `max_count` | `null` | Optional inclusive upper bound. When set, the criterion passes iff `min_count <= matches <= max_count`. |
+| `require_success` | `false` | Only count commands that completed successfully. |
+| `exclude_pattern` | `null` | Regex that must NOT match; a command matching both `command_pattern` and `exclude_pattern` is skipped. Also matched with shell normalization (see below). |
+
+**Shell normalization.** For a Bash command, both `command_pattern` and `exclude_pattern` are matched against the raw command text **and** its shell-normalized form — the `bash`/`sh`/`zsh -lc "..."` wrapper stripped and shell quoting resolved with `shlex` — and a hit on *either* form counts. So a pattern like `curated_channels` matches whether the agent wrote the argument bare, `'single'`-quoted, `"double"`-quoted, or `\"escaped\"`; you do **not** hand-encode shell quoting. Because the same haystacks also feed `exclude_pattern` and the `max_count` gate, normalization is **not** purely additive: a quote-obfuscated call can now be caught by an exclusion or a `max_count: 0` gate that the raw text alone would have missed — and, conversely, an unedited `exclude_pattern` may now exclude a call it previously let through. Cross-repo suites that hand-encoded quote tolerance in their patterns should re-baseline.
+
 **Codex limitation.** Codex agents map `Read`, `Grep`, and `Glob` tools to `shell` commands (they execute via bash), so `tool_name: "Read"` on Codex returns no matches. Use `tool_name: "Bash"` or `tool_name: null` (any tool) for Codex-compatible checks. This criterion works correctly on Claude Code agents, which emit separate `Read`/`Grep`/`Glob` telemetry.
 
 ### `commands_efficiency`
@@ -891,6 +909,8 @@ Have an LLM grade the task against a rubric written in the task YAML. **Continuo
 | `temperature` | `0.0` | Sampling temperature (0.0 = deterministic) |
 | `max_tokens` | `2000` | Maximum tokens in the judge's response |
 | `max_file_chars` | `20000` | Per-file (and agent_output) truncation applied before building the prompt |
+| `capture_transcript` | `true` | Persist a `JudgeTranscript` (raw verdict + rendered prompts + token usage) to a sibling `judge-<idx>.yaml`. Set `false` to drop it when on-disk size matters (e.g. 1000-row datasets); the `findings` on the result persist regardless. |
+| `max_transcript_chars` | `100000` | Aggregate cap on captured transcript text (verdict + prompt + system, split 60/30/10). Exceeding it marks the transcript `truncated=True`. |
 
 **Transport selection.** The judge call is routed by the active `API_BACKEND`:
 
@@ -964,6 +984,8 @@ Spawn a full Claude Code SDK agent as the judge. Unlike `llm_judge` (a single LL
 | `max_turns` | `50` | Judge's inner-loop turn limit |
 | `turn_timeout` | `300` | Wall-clock timeout (seconds) |
 | `agent` | hardened judge defaults | Nested `AgentConfig` — `model`, `permission_mode`, `allowed_tools`, `disallowed_tools`, `ignore_patterns`, `sdk_options`. A partial block (e.g. only `model:`) still applies the judge security defaults for missing fields, and the security floor (`.claude` / `.mcp.json` / `_reference` ignore patterns, `setting_sources=[]`) is always enforced. |
+| `capture_transcript` | `true` | Persist a `JudgeTranscript` (tool calls + token usage + raw verdict + rendered prompts) to a sibling `judge-<idx>.yaml`. Set `false` to drop the trajectory log when on-disk size matters; the `findings` on the result persist regardless. |
+| `max_transcript_chars` | `100000` | Aggregate cap on captured transcript text (verdict + prompt + system + tool detail/result-preview lines, split 60/30/10 with tool calls prioritized). Exceeding it marks the transcript `truncated=True`. |
 
 **Security**
 
diff --git a/src/coder_eval/criteria/command_executed.py b/src/coder_eval/criteria/command_executed.py
index 2cb0e746..69aad136 100644
--- a/src/coder_eval/criteria/command_executed.py
+++ b/src/coder_eval/criteria/command_executed.py
@@ -60,12 +60,20 @@ def _normalize_shell(cmd_text: str) -> str | None:
     the pattern. Authors then hand-model that escaping and get it subtly wrong
     (e.g. allowing ``"`` but not ``'``), silently under-counting correct calls.
 
-    This unwraps a ``bash``/``sh -c`` wrapper and resolves shell quoting with
-    ``shlex`` so a pattern can match argv semantics regardless of quoting.
+    This unwraps a ``bash``/``sh``/``zsh -c`` wrapper and resolves shell quoting
+    with ``shlex`` so a pattern can match argv semantics regardless of quoting.
     Shell operators (``&&``, ``|``, ``>``) survive as their own tokens, so
-    patterns that reference them keep working. Returns ``None`` when the text
-    can't be parsed (unbalanced quotes, heredocs); the caller keeps the raw
-    text as a haystack, so nothing that matched before can stop matching.
+    patterns that reference them keep working, and embedded newlines collapse to
+    single spaces. Returns ``None`` when the text can't be parsed (an odd quote
+    count — NOT heredocs, which tokenize fine); the caller then keeps only the
+    raw text as a haystack.
+
+    Adding a second (normalized) haystack is *not* purely additive: a
+    ``command_pattern`` can only gain matches, but the same haystacks feed
+    ``exclude_pattern`` and the ``max_count`` gate, so a normalized form can
+    newly satisfy an exclusion or trip a ``max_count`` cap — i.e. a command that
+    counted on the raw text alone can stop counting. See
+    ``CommandExecutedChecker._matching_commands``.
 
     Memoized (pure function of ``cmd_text``): the early-stop watcher re-scans the
     whole accumulated trajectory on every tool-call event, so the same command is
diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py
index 26a1142c..baa2742b 100644
--- a/src/coder_eval/models/criteria.py
+++ b/src/coder_eval/models/criteria.py
@@ -593,7 +593,14 @@ class CommandExecutedCriterion(LiveSuccessCriterion):
     type: Literal["command_executed"] = "command_executed"
     tool_name: str | None = Field(default=None, description="Tool name filter (e.g., 'Bash'). None = any tool.")
     command_pattern: str | None = Field(
-        default=None, description="Regex to match command parameters. None = any command."
+        default=None,
+        description=(
+            "Regex to match command parameters. None = any command. For a Bash "
+            "command the pattern is matched against the raw command text OR its "
+            "shell-normalized form — `shlex`-resolved quoting with a "
+            "`bash`/`zsh -lc` wrapper stripped — whichever hits, so you need not "
+            "hand-encode shell quoting/escaping (`'single'`, `\"double\"`, bare)."
+        ),
     )
     min_count: int = Field(
         default=1,
@@ -617,7 +624,9 @@ class CommandExecutedCriterion(LiveSuccessCriterion):
     exclude_pattern: str | None = Field(
         default=None,
         description=(
-            "Regex that must NOT match. Commands matching both command_pattern and exclude_pattern are skipped."
+            "Regex that must NOT match. Commands matching both command_pattern and exclude_pattern are skipped. "
+            "Like command_pattern, this is matched against the raw Bash command OR its shell-normalized form, so "
+            "it also excludes quote-obfuscated variants of the same call."
         ),
     )
 
diff --git a/tests/lint/doc_schema_parity.py b/tests/lint/doc_schema_parity.py
index 5ded814d..8b981446 100644
--- a/tests/lint/doc_schema_parity.py
+++ b/tests/lint/doc_schema_parity.py
@@ -12,10 +12,16 @@ class impossible to reintroduce: for a small, explicit registry of user-facing
 * **Allowlist, not denylist.** A new field on a registered model that is neither
   documented nor exempted *fails* — which is the point. Adding a user-facing field
   now forces a doc update or a reasoned exemption in the same change.
-* **Explicit registry, no recursion.** Only the four registered models are
-  checked; nested models (``AgentConfig``, ``SandboxConfig``, criteria, …) are NOT
-  walked. Walking them would silently expand the documentation commitment to
-  dozens of models nobody signed up for.
+* **Registry + the criterion union, no recursion.** The four top-level models are
+  registered explicitly; the members of the ``SuccessCriterion`` discriminated
+  union are enumerated programmatically (a task author writes them directly, the
+  guide already claims to be their reference, and the set is closed/enumerable).
+  In neither case do we recurse into *nested* models (``AgentConfig``,
+  ``SandboxConfig``, the criteria's own sub-models, …) — that would silently
+  expand the commitment to dozens of models nobody signed up for. Enumerating the
+  union (not recursing) is what makes "the next criterion-semantics change cannot
+  ship undocumented" mechanical: a new criterion, or a new field on one, fails
+  until it is documented or exempted.
 * **Inline-code match, deliberately simple.** A field counts as documented when
   its bare name appears wrapped in Markdown inline-code backticks anywhere in the
   doc. This is a floor, not a proof — a field name that appears in an unrelated
@@ -32,20 +38,48 @@ class impossible to reintroduce: for a small, explicit registry of user-facing
 from __future__ import annotations
 
 from pathlib import Path
+from typing import get_args
 
 from pydantic import BaseModel
 
 from coder_eval.models import Dataset, RunLimits, SimulationConfig, TaskDefinition
+from coder_eval.models.criteria import SuccessCriterion
+
+
+_GUIDE = "docs/TASK_DEFINITION_GUIDE.md"
+
+
+def _criterion_models() -> list[type[BaseModel]]:
+    """Flatten the ``SuccessCriterion`` discriminated union into its member models.
+
+    Enumerates the union members (``FileExistsCriterion``, ``CommandExecutedCriterion``,
+    …) without recursing into each member's own nested models, so the documentation
+    promise covers every criterion a task author can write — and any newly added
+    criterion automatically — while staying bounded to the union itself.
+    """
+    seen: list[type[BaseModel]] = []
+
+    def walk(tp: object) -> None:
+        for arg in get_args(tp):
+            if isinstance(arg, type) and issubclass(arg, BaseModel):
+                if arg not in seen:
+                    seen.append(arg)
+            else:
+                walk(arg)
+
+    walk(SuccessCriterion)
+    return seen
 
 
 # Models the project commits to documenting, paired with the doc page that owns
-# their field reference. Keep this list SHORT and explicit — every entry is a
-# standing documentation obligation.
+# their field reference. The four top-level models are explicit; the criterion
+# union members are enumerated so a new criterion (or field) can't ship undocumented.
 DOCUMENTED_MODELS: list[tuple[type[BaseModel], str]] = [
-    (TaskDefinition, "docs/TASK_DEFINITION_GUIDE.md"),
-    (RunLimits, "docs/TASK_DEFINITION_GUIDE.md"),
-    (Dataset, "docs/TASK_DEFINITION_GUIDE.md"),
-    (SimulationConfig, "docs/TASK_DEFINITION_GUIDE.md"),
+    (TaskDefinition, _GUIDE),
+    (RunLimits, _GUIDE),
+    (Dataset, _GUIDE),
+    (SimulationConfig, _GUIDE),
+    *[(m, _GUIDE) for m in _criterion_models()],
 ]
 
 # Fields deliberately absent from the user docs, with the reason each is not

From 4486584ee458d4fa2dadff6025c9dfa18102981c Mon Sep 17 00:00:00 2001
From: rockymadden <git@rockymadden.com>
Date: Wed, 5 Aug 2026 09:01:41 -0600
Subject: [PATCH 07/10] fix(lint): scope CE030 criterion parity to in-tree
 criteria only
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The CE030 extension flattened the whole `SuccessCriterion` union, but a plugin
can inject its own criterion into that union at load time via the
`coder_eval.plugins` hook — the `uipath` SDK (installed by CI's `--extra uipath`)
adds a `CliCalledCriterion` (fields `log`/`positional`). CE030 then demanded the
in-repo guide document a criterion this repo doesn't own, failing CI's Quality
Gate (the failure did not reproduce on macOS, whose lockfile resolution omits
the injecting package).

Scope `_criterion_models()` to criteria defined in `coder_eval.models.criteria`
(filter on `__module__`), and read the union live from the module so plugin
reassignment before import is still handled. Plugin-contributed criteria are
documented in their own repos, not this guide.

Adds two regression tests: the enumerated members are all in-tree, and a
plugin-injected (foreign-`__module__`) criterion is excluded from the parity
check.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 tests/lint/doc_schema_parity.py | 22 ++++++++++++++-----
 tests/test_custom_lint.py       | 38 +++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/tests/lint/doc_schema_parity.py b/tests/lint/doc_schema_parity.py
index 8b981446..1cd5500a 100644
--- a/tests/lint/doc_schema_parity.py
+++ b/tests/lint/doc_schema_parity.py
@@ -43,31 +43,43 @@ class impossible to reintroduce: for a small, explicit registry of user-facing
 from pydantic import BaseModel
 
 from coder_eval.models import Dataset, RunLimits, SimulationConfig, TaskDefinition
-from coder_eval.models.criteria import SuccessCriterion
+from coder_eval.models import criteria as _criteria_module
 
 
 _GUIDE = "docs/TASK_DEFINITION_GUIDE.md"
 
+# Module that owns the in-tree criteria. A plugin (e.g. the `uipath` SDK's
+# `coder_eval.plugins` hook) can inject its own criterion into the
+# ``SuccessCriterion`` union at load time; such a criterion is defined in the
+# plugin's package, is documented in the plugin's own repo, and must NOT impose a
+# doc obligation on THIS repo's guide. We therefore scope the parity check to
+# criteria defined here.
+_IN_TREE_CRITERIA_MODULE = _criteria_module.__name__
+
 
 def _criterion_models() -> list[type[BaseModel]]:
-    """Flatten the ``SuccessCriterion`` discriminated union into its member models.
+    """Flatten the ``SuccessCriterion`` union into its **in-tree** member models.
 
     Enumerates the union members (``FileExistsCriterion``, ``CommandExecutedCriterion``,
     …) without recursing into each member's own nested models, so the documentation
     promise covers every criterion a task author can write — and any newly added
-    criterion automatically — while staying bounded to the union itself.
+    in-tree criterion automatically — while staying bounded to the union itself.
+    Plugin-contributed criteria (``__module__`` outside ``coder_eval.models.criteria``)
+    are excluded: they belong to their plugin's docs, not this guide.
     """
     seen: list[type[BaseModel]] = []
 
     def walk(tp: object) -> None:
         for arg in get_args(tp):
             if isinstance(arg, type) and issubclass(arg, BaseModel):
-                if arg not in seen:
+                if arg not in seen and arg.__module__ == _IN_TREE_CRITERIA_MODULE:
                     seen.append(arg)
             else:
                 walk(arg)
 
-    walk(SuccessCriterion)
+    # Read the union live from the module (not an import-time-bound copy) so a
+    # plugin that reassigns it before this runs is still handled by the filter.
+    walk(_criteria_module.SuccessCriterion)
     return seen
 
 
diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py
index a1f4da70..c78d05ff 100644
--- a/tests/test_custom_lint.py
+++ b/tests/test_custom_lint.py
@@ -910,6 +910,44 @@ def test_registered_models_are_fully_documented(self):
             + "\n".join(f"  {model}: {', '.join(fields)}" for model, fields in sorted(findings.items()))
         )
 
+    def test_criterion_models_cover_the_in_tree_union(self):
+        # The criterion members must all come from the in-tree criteria module,
+        # and the set must be non-empty (guards against the flatten silently
+        # returning nothing after a refactor).
+        from tests.lint.doc_schema_parity import _IN_TREE_CRITERIA_MODULE, _criterion_models
+
+        models = _criterion_models()
+        assert models, "expected the SuccessCriterion union to yield in-tree criterion models"
+        assert all(m.__module__ == _IN_TREE_CRITERIA_MODULE for m in models)
+
+    def test_plugin_contributed_criterion_is_excluded(self, monkeypatch):
+        """A criterion a plugin injects into the union (foreign ``__module__``) is
+        NOT held to this repo's doc obligation.
+
+        Regression: the ``uipath`` SDK's ``coder_eval.plugins`` hook adds a
+        ``CliCalledCriterion`` (fields ``log``/``positional``) to the union in CI;
+        the first CE030-over-the-union attempt wrongly demanded the in-repo guide
+        document it. The doc promise is scoped to in-tree criteria only.
+        """
+        from typing import Annotated, Literal
+
+        from pydantic import Field
+
+        from coder_eval.models import criteria as crit
+        from tests.lint.doc_schema_parity import _criterion_models
+
+        class _PluginCriterion(crit.BaseSuccessCriterion):
+            type: Literal["_plugin_only"] = "_plugin_only"
+            undocumented_plugin_field: str = ""
+
+        _PluginCriterion.__module__ = "some_plugin.criteria"  # not the in-tree module
+        extended = Annotated[crit.FileExistsCriterion | _PluginCriterion, Field(discriminator="type")]
+        monkeypatch.setattr(crit, "SuccessCriterion", extended)
+
+        names = [m.__name__ for m in _criterion_models()]
+        assert "FileExistsCriterion" in names
+        assert "_PluginCriterion" not in names
+
     def test_exempt_fields_carry_a_reason(self):
         from tests.lint.doc_schema_parity import EXEMPT
 

From 585eb8397d604700c31d4789ef8682d89f6a0963 Mon Sep 17 00:00:00 2001
From: rockymadden <git@rockymadden.com>
Date: Wed, 5 Aug 2026 09:09:39 -0600
Subject: [PATCH 08/10] fix(lint): enumerate in-tree criteria by module
 attribute, not the union

The previous CE030 scope fix filtered union members by ``__module__``, but the
injected ``CliCalledCriterion`` is built with ``pydantic.create_model`` and a
spoofed ``__module__="coder_eval.models.criteria"``, so the string filter did
not exclude it and CI still failed.

Enumerate the concrete ``BaseSuccessCriterion`` subclasses that are genuine
module-level attributes of ``coder_eval.models.criteria`` instead of flattening
the ``SuccessCriterion`` union at all. A ``create_model`` class is not inserted
into the module namespace, so union injection can't contaminate the set, while
every criterion this repo actually defines is still covered automatically.

The regression test now mirrors the real shape (create_model + spoofed
``__module__`` + union splice, never a module attribute) and asserts exclusion.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 tests/lint/doc_schema_parity.py | 54 +++++++++++++++------------------
 tests/test_custom_lint.py       | 53 +++++++++++++++++++-------------
 2 files changed, 57 insertions(+), 50 deletions(-)

diff --git a/tests/lint/doc_schema_parity.py b/tests/lint/doc_schema_parity.py
index 1cd5500a..cde6de47 100644
--- a/tests/lint/doc_schema_parity.py
+++ b/tests/lint/doc_schema_parity.py
@@ -38,49 +38,45 @@ class impossible to reintroduce: for a small, explicit registry of user-facing
 from __future__ import annotations
 
 from pathlib import Path
-from typing import get_args
 
 from pydantic import BaseModel
 
 from coder_eval.models import Dataset, RunLimits, SimulationConfig, TaskDefinition
 from coder_eval.models import criteria as _criteria_module
+from coder_eval.models.criteria import BaseSuccessCriterion, LiveSuccessCriterion
 
 
 _GUIDE = "docs/TASK_DEFINITION_GUIDE.md"
 
-# Module that owns the in-tree criteria. A plugin (e.g. the `uipath` SDK's
-# `coder_eval.plugins` hook) can inject its own criterion into the
-# ``SuccessCriterion`` union at load time; such a criterion is defined in the
-# plugin's package, is documented in the plugin's own repo, and must NOT impose a
-# doc obligation on THIS repo's guide. We therefore scope the parity check to
-# criteria defined here.
+# Module that owns the in-tree criteria.
 _IN_TREE_CRITERIA_MODULE = _criteria_module.__name__
 
 
 def _criterion_models() -> list[type[BaseModel]]:
-    """Flatten the ``SuccessCriterion`` union into its **in-tree** member models.
-
-    Enumerates the union members (``FileExistsCriterion``, ``CommandExecutedCriterion``,
-    …) without recursing into each member's own nested models, so the documentation
-    promise covers every criterion a task author can write — and any newly added
-    in-tree criterion automatically — while staying bounded to the union itself.
-    Plugin-contributed criteria (``__module__`` outside ``coder_eval.models.criteria``)
-    are excluded: they belong to their plugin's docs, not this guide.
+    """The criterion models defined in ``coder_eval.models.criteria``.
+
+    Enumerates the concrete ``BaseSuccessCriterion`` subclasses that are genuine
+    module-level attributes of the criteria module — so any new in-tree criterion
+    is covered automatically — rather than flattening the ``SuccessCriterion``
+    union. A plugin's ``coder_eval.plugins`` hook can inject its OWN criterion into
+    that union at load time (the ``uipath`` SDK adds a ``CliCalledCriterion``, built
+    with ``pydantic.create_model`` and a spoofed ``__module__`` so it even reports
+    as living here), and such a criterion — documented in the plugin's own repo —
+    must not impose a doc obligation on THIS guide. A ``create_model`` class is not
+    inserted into the module namespace, so enumerating real module attributes (not
+    the union) excludes it while still covering every criterion this repo defines.
     """
-    seen: list[type[BaseModel]] = []
-
-    def walk(tp: object) -> None:
-        for arg in get_args(tp):
-            if isinstance(arg, type) and issubclass(arg, BaseModel):
-                if arg not in seen and arg.__module__ == _IN_TREE_CRITERIA_MODULE:
-                    seen.append(arg)
-            else:
-                walk(arg)
-
-    # Read the union live from the module (not an import-time-bound copy) so a
-    # plugin that reassigns it before this runs is still handled by the filter.
-    walk(_criteria_module.SuccessCriterion)
-    return seen
+    out: list[type[BaseModel]] = []
+    for obj in vars(_criteria_module).values():
+        if (
+            isinstance(obj, type)
+            and issubclass(obj, BaseSuccessCriterion)
+            and obj not in (BaseSuccessCriterion, LiveSuccessCriterion)  # abstract bases
+            and obj.__module__ == _IN_TREE_CRITERIA_MODULE  # not a re-export
+            and obj not in out
+        ):
+            out.append(obj)
+    return out
 
 
 # Models the project commits to documenting, paired with the doc page that owns
diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py
index c78d05ff..c7719ddf 100644
--- a/tests/test_custom_lint.py
+++ b/tests/test_custom_lint.py
@@ -910,43 +910,54 @@ def test_registered_models_are_fully_documented(self):
             + "\n".join(f"  {model}: {', '.join(fields)}" for model, fields in sorted(findings.items()))
         )
 
-    def test_criterion_models_cover_the_in_tree_union(self):
-        # The criterion members must all come from the in-tree criteria module,
-        # and the set must be non-empty (guards against the flatten silently
-        # returning nothing after a refactor).
+    def test_criterion_models_are_the_in_tree_criteria(self):
+        # Every enumerated criterion is a real module attribute of the in-tree
+        # criteria module, the set is non-empty, and it matches the union literal
+        # written in criteria.py (so this can't silently under- or over-count).
+        from coder_eval.models import criteria as crit
         from tests.lint.doc_schema_parity import _IN_TREE_CRITERIA_MODULE, _criterion_models
 
         models = _criterion_models()
-        assert models, "expected the SuccessCriterion union to yield in-tree criterion models"
+        assert models, "expected in-tree criterion models"
         assert all(m.__module__ == _IN_TREE_CRITERIA_MODULE for m in models)
+        assert all(getattr(crit, m.__name__, None) is m for m in models)
 
-    def test_plugin_contributed_criterion_is_excluded(self, monkeypatch):
-        """A criterion a plugin injects into the union (foreign ``__module__``) is
-        NOT held to this repo's doc obligation.
+    def test_plugin_injected_criterion_is_excluded(self, monkeypatch):
+        """A criterion a plugin injects into the union is NOT held to this repo's
+        doc obligation — even when it spoofs ``__module__`` to look in-tree.
 
         Regression: the ``uipath`` SDK's ``coder_eval.plugins`` hook adds a
-        ``CliCalledCriterion`` (fields ``log``/``positional``) to the union in CI;
-        the first CE030-over-the-union attempt wrongly demanded the in-repo guide
-        document it. The doc promise is scoped to in-tree criteria only.
+        ``CliCalledCriterion`` (fields ``log``/``positional``) to the union in CI,
+        built via ``create_model`` with ``__module__="coder_eval.models.criteria"``,
+        so a ``__module__``-string filter did not exclude it. Enumerating genuine
+        module attributes (not the union) does: a ``create_model`` class is not
+        inserted into the module namespace. This pins that behaviour.
         """
         from typing import Annotated, Literal
 
-        from pydantic import Field
+        from pydantic import Field, create_model
 
         from coder_eval.models import criteria as crit
         from tests.lint.doc_schema_parity import _criterion_models
 
-        class _PluginCriterion(crit.BaseSuccessCriterion):
-            type: Literal["_plugin_only"] = "_plugin_only"
-            undocumented_plugin_field: str = ""
-
-        _PluginCriterion.__module__ = "some_plugin.criteria"  # not the in-tree module
-        extended = Annotated[crit.FileExistsCriterion | _PluginCriterion, Field(discriminator="type")]
-        monkeypatch.setattr(crit, "SuccessCriterion", extended)
+        # Mirror the plugin's shape: __module__ spoofed to the in-tree module, and
+        # spliced into the union — but never assigned as a module attribute.
+        injected = create_model(
+            "CliCalledCriterionStub",
+            __base__=crit.BaseSuccessCriterion,
+            __module__=crit.__name__,
+            type=(Literal["_cli_called_stub"], "_cli_called_stub"),
+            undocumented_plugin_field=(str, ""),
+        )
+        monkeypatch.setattr(
+            crit,
+            "SuccessCriterion",
+            Annotated[crit.FileExistsCriterion | injected, Field(discriminator="type")],
+        )
 
         names = [m.__name__ for m in _criterion_models()]
-        assert "FileExistsCriterion" in names
-        assert "_PluginCriterion" not in names
+        assert "FileExistsCriterion" in names  # a genuine module attribute
+        assert "CliCalledCriterionStub" not in names  # union-injected, not a module attr
 
     def test_exempt_fields_carry_a_reason(self):
         from tests.lint.doc_schema_parity import EXEMPT

From a6d9a550d25478a6f7afeeb99e75b5b5c22e7a84 Mon Sep 17 00:00:00 2001
From: rockymadden <git@rockymadden.com>
Date: Wed, 5 Aug 2026 09:14:15 -0600
Subject: [PATCH 09/10] fix(lint): derive CE030 criteria from the source union
 literal, not runtime
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The `uipath` SDK's `coder_eval.plugins` hook integrates its `CliCalledCriterion`
so thoroughly at load time — real module attribute, `__module__` spoofed to
`coder_eval.models.criteria`, and spliced into the runtime `SuccessCriterion`
union — that neither a `__module__` filter nor a module-attribute scan could
tell it apart from an in-tree criterion. Every runtime signal is contaminated.

Read the union members from the `SuccessCriterion = Annotated[...]` literal in
the criteria.py SOURCE (AST parse), then resolve each name via getattr. A plugin
cannot edit this repo's source, so the list is exactly the criteria this repo
ships — and a new in-tree criterion is still covered automatically (it's added
to the source literal). The regression test now reproduces the full injection
shape (module attr + union splice + spoofed __module__) and asserts exclusion.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 tests/lint/doc_schema_parity.py | 57 ++++++++++++++++++---------------
 tests/test_custom_lint.py       | 39 +++++++++++-----------
 2 files changed, 51 insertions(+), 45 deletions(-)

diff --git a/tests/lint/doc_schema_parity.py b/tests/lint/doc_schema_parity.py
index cde6de47..91491015 100644
--- a/tests/lint/doc_schema_parity.py
+++ b/tests/lint/doc_schema_parity.py
@@ -37,46 +37,51 @@ class impossible to reintroduce: for a small, explicit registry of user-facing
 
 from __future__ import annotations
 
+import ast
 from pathlib import Path
 
 from pydantic import BaseModel
 
 from coder_eval.models import Dataset, RunLimits, SimulationConfig, TaskDefinition
 from coder_eval.models import criteria as _criteria_module
-from coder_eval.models.criteria import BaseSuccessCriterion, LiveSuccessCriterion
 
 
 _GUIDE = "docs/TASK_DEFINITION_GUIDE.md"
 
-# Module that owns the in-tree criteria.
-_IN_TREE_CRITERIA_MODULE = _criteria_module.__name__
 
+def _union_literal_criterion_names() -> list[str]:
+    """Class names in the ``SuccessCriterion = Annotated[A | B | ...]`` literal.
 
-def _criterion_models() -> list[type[BaseModel]]:
-    """The criterion models defined in ``coder_eval.models.criteria``.
-
-    Enumerates the concrete ``BaseSuccessCriterion`` subclasses that are genuine
-    module-level attributes of the criteria module — so any new in-tree criterion
-    is covered automatically — rather than flattening the ``SuccessCriterion``
-    union. A plugin's ``coder_eval.plugins`` hook can inject its OWN criterion into
-    that union at load time (the ``uipath`` SDK adds a ``CliCalledCriterion``, built
-    with ``pydantic.create_model`` and a spoofed ``__module__`` so it even reports
-    as living here), and such a criterion — documented in the plugin's own repo —
-    must not impose a doc obligation on THIS guide. A ``create_model`` class is not
-    inserted into the module namespace, so enumerating real module attributes (not
-    the union) excludes it while still covering every criterion this repo defines.
+    Read STATICALLY from ``criteria.py`` source, not from the runtime union
+    object. That is the whole point: a plugin's ``coder_eval.plugins`` hook can
+    inject its OWN criterion into the runtime union (and even into the module
+    namespace) at load time — the ``uipath`` SDK adds a ``CliCalledCriterion`` —
+    and such a criterion is documented in its plugin's own repo, not this guide.
+    A plugin cannot edit this repo's source, so the source literal is the
+    authoritative, contamination-proof list of the criteria THIS repo ships.
     """
-    out: list[type[BaseModel]] = []
-    for obj in vars(_criteria_module).values():
-        if (
-            isinstance(obj, type)
-            and issubclass(obj, BaseSuccessCriterion)
-            and obj not in (BaseSuccessCriterion, LiveSuccessCriterion)  # abstract bases
-            and obj.__module__ == _IN_TREE_CRITERIA_MODULE  # not a re-export
-            and obj not in out
+    tree = ast.parse(Path(_criteria_module.__file__).read_text(encoding="utf-8"))
+    for node in tree.body:
+        if isinstance(node, ast.Assign) and any(
+            isinstance(t, ast.Name) and t.id == "SuccessCriterion" for t in node.targets
         ):
-            out.append(obj)
-    return out
+            # value is ``Annotated[<Name | Name | ...>, Field(...)]``; every
+            # criterion class name ends in "Criterion" (Annotated/Field/discriminator
+            # string do not), so this selects exactly the union members.
+            return [n.id for n in ast.walk(node.value) if isinstance(n, ast.Name) and n.id.endswith("Criterion")]
+    raise AssertionError("SuccessCriterion union assignment not found in criteria.py")
+
+
+def _criterion_models() -> list[type[BaseModel]]:
+    """The in-tree ``SuccessCriterion`` member models, resolved from the source union.
+
+    Names come from the source literal (see :func:`_union_literal_criterion_names`),
+    then each is resolved to its class via ``getattr`` on the criteria module. New
+    in-tree criteria are covered automatically (they're added to the union literal);
+    plugin-injected criteria are never named in the source, so they are excluded
+    even though they exist as runtime attributes.
+    """
+    return [getattr(_criteria_module, name) for name in _union_literal_criterion_names()]
 
 
 # Models the project commits to documenting, paired with the doc page that owns
diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py
index c7719ddf..bba7ba31 100644
--- a/tests/test_custom_lint.py
+++ b/tests/test_custom_lint.py
@@ -910,28 +910,29 @@ def test_registered_models_are_fully_documented(self):
             + "\n".join(f"  {model}: {', '.join(fields)}" for model, fields in sorted(findings.items()))
         )
 
-    def test_criterion_models_are_the_in_tree_criteria(self):
-        # Every enumerated criterion is a real module attribute of the in-tree
-        # criteria module, the set is non-empty, and it matches the union literal
-        # written in criteria.py (so this can't silently under- or over-count).
+    def test_criterion_models_match_the_source_union_literal(self):
+        # The enumerated criteria are exactly the class names in the source
+        # ``SuccessCriterion`` union literal, resolved to real module attributes.
         from coder_eval.models import criteria as crit
-        from tests.lint.doc_schema_parity import _IN_TREE_CRITERIA_MODULE, _criterion_models
+        from tests.lint.doc_schema_parity import _criterion_models, _union_literal_criterion_names
 
+        names = _union_literal_criterion_names()
+        assert names, "expected the source union literal to name criteria"
         models = _criterion_models()
-        assert models, "expected in-tree criterion models"
-        assert all(m.__module__ == _IN_TREE_CRITERIA_MODULE for m in models)
+        assert [m.__name__ for m in models] == names
         assert all(getattr(crit, m.__name__, None) is m for m in models)
 
     def test_plugin_injected_criterion_is_excluded(self, monkeypatch):
-        """A criterion a plugin injects into the union is NOT held to this repo's
-        doc obligation — even when it spoofs ``__module__`` to look in-tree.
+        """A criterion a plugin injects at load time is NOT held to this repo's
+        doc obligation — even when it is a real module attribute, spoofs
+        ``__module__`` to look in-tree, AND is spliced into the runtime union.
 
         Regression: the ``uipath`` SDK's ``coder_eval.plugins`` hook adds a
-        ``CliCalledCriterion`` (fields ``log``/``positional``) to the union in CI,
-        built via ``create_model`` with ``__module__="coder_eval.models.criteria"``,
-        so a ``__module__``-string filter did not exclude it. Enumerating genuine
-        module attributes (not the union) does: a ``create_model`` class is not
-        inserted into the module namespace. This pins that behaviour.
+        ``CliCalledCriterion`` (fields ``log``/``positional``) in CI, integrated
+        so thoroughly that neither a ``__module__`` filter nor a module-attribute
+        scan excluded it. The parity list is read from the SOURCE union literal,
+        which a plugin cannot edit, so runtime injection has no effect. This
+        reproduces the full shape and pins that.
         """
         from typing import Annotated, Literal
 
@@ -940,15 +941,15 @@ def test_plugin_injected_criterion_is_excluded(self, monkeypatch):
         from coder_eval.models import criteria as crit
         from tests.lint.doc_schema_parity import _criterion_models
 
-        # Mirror the plugin's shape: __module__ spoofed to the in-tree module, and
-        # spliced into the union — but never assigned as a module attribute.
         injected = create_model(
             "CliCalledCriterionStub",
             __base__=crit.BaseSuccessCriterion,
-            __module__=crit.__name__,
+            __module__=crit.__name__,  # spoof: looks in-tree
             type=(Literal["_cli_called_stub"], "_cli_called_stub"),
             undocumented_plugin_field=(str, ""),
         )
+        # Integrate it exactly as CI's plugin does: real module attribute + union splice.
+        monkeypatch.setattr(crit, "CliCalledCriterionStub", injected, raising=False)
         monkeypatch.setattr(
             crit,
             "SuccessCriterion",
@@ -956,8 +957,8 @@ def test_plugin_injected_criterion_is_excluded(self, monkeypatch):
         )
 
         names = [m.__name__ for m in _criterion_models()]
-        assert "FileExistsCriterion" in names  # a genuine module attribute
-        assert "CliCalledCriterionStub" not in names  # union-injected, not a module attr
+        assert "FileExistsCriterion" in names  # named in the source union literal
+        assert "CliCalledCriterionStub" not in names  # injected at runtime, not in source
 
     def test_exempt_fields_carry_a_reason(self):
         from tests.lint.doc_schema_parity import EXEMPT

From d8a77ea9361305199a534ac875f9bd03dbd6b609 Mon Sep 17 00:00:00 2001
From: rockymadden <git@rockymadden.com>
Date: Wed, 5 Aug 2026 09:19:52 -0600
Subject: [PATCH 10/10] revert(lint): drop CE030 SuccessCriterion-union
 extension (env-hostile in CI)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CI installs `--extra uipath`, and in that environment `coder_eval.models.criteria`
gains a `CliCalledCriterion` (fields `log`/`positional`) absent from a plain
checkout — it did not reproduce on macOS. It defeated every way tried to tell it
apart from an in-tree criterion: union membership, a `__module__` string filter
(spoofed to the in-tree module), a genuine-module-attribute scan (it is setattr'd
onto the module), and an AST parse of the union literal in criteria.py source (CI's
criteria module resolves to a file whose literal already contains it).

Restore CE030 to the original four top-level models. The `command_pattern` /
`exclude_pattern` contract this PR changed remains documented in the Field
descriptions and TASK_DEFINITION_GUIDE; the union extension is parked in
.claude/harness-candidates.md with the full diagnosis for a future revisit.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .claude/harness-candidates.md   | 20 ++++++++++
 tests/lint/doc_schema_parity.py | 67 +++++----------------------------
 tests/test_custom_lint.py       | 50 ------------------------
 3 files changed, 30 insertions(+), 107 deletions(-)

diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md
index dd3912db..8bf1b8af 100644
--- a/.claude/harness-candidates.md
+++ b/.claude/harness-candidates.md
@@ -107,3 +107,23 @@ Deferred lint/test guardrails surfaced during reviews. Promote to a `CExxx` rule
   caught them. The cleanup plan explicitly deferred this as YAGNI for the
   one-time purge, but any future doc rename/deletion re-opens the same blind
   spot — caught in the 2026-07-03 open-source-docs-cleanup implementation run.
+
+## From PR #77 (command-executed shell-normalize) — CE030-to-criteria deferred
+
+- [ ] **Extend CE030 doc/schema-parity to the `SuccessCriterion` union** so a new
+  criterion (or field) can't ship undocumented. Attempted in PR #77 and reverted:
+  CI installs `--extra uipath`, and in that environment `coder_eval.models.criteria`
+  gains a `CliCalledCriterion` (fields `log`/`positional`) that is NOT present in a
+  plain checkout (it did not reproduce on macOS, whose lockfile resolution omits the
+  contributing linux-only component). It defeated every discriminator tried — union
+  membership, a `__module__` string filter (it is spoofed to `coder_eval.models.criteria`),
+  a genuine-module-attribute scan (it is `setattr` onto the module), and even an AST
+  parse of the `SuccessCriterion` union literal in `criteria.py` source (CI's imported
+  criteria module resolves to a file whose union literal already contains it). No
+  runtime OR source signal available in the lint could separate the injected criterion
+  from an in-tree one. Revisit only with a way to identify the in-tree criterion set that
+  is provably immune to the uipath integration — e.g. a hardcoded name allowlist of the
+  in-tree criteria (losing auto-coverage of new ones), or first understanding exactly how
+  that environment injects the criterion. Until then CE030 stays scoped to the four
+  top-level models; the `command_pattern`/`exclude_pattern` contract this PR changed is
+  documented in the Field descriptions and TASK_DEFINITION_GUIDE regardless.
diff --git a/tests/lint/doc_schema_parity.py b/tests/lint/doc_schema_parity.py
index 91491015..5ded814d 100644
--- a/tests/lint/doc_schema_parity.py
+++ b/tests/lint/doc_schema_parity.py
@@ -12,16 +12,10 @@ class impossible to reintroduce: for a small, explicit registry of user-facing
 * **Allowlist, not denylist.** A new field on a registered model that is neither
   documented nor exempted *fails* — which is the point. Adding a user-facing field
   now forces a doc update or a reasoned exemption in the same change.
-* **Registry + the criterion union, no recursion.** The four top-level models are
-  registered explicitly; the members of the ``SuccessCriterion`` discriminated
-  union are enumerated programmatically (a task author writes them directly, the
-  guide already claims to be their reference, and the set is closed/enumerable).
-  In neither case do we recurse into *nested* models (``AgentConfig``,
-  ``SandboxConfig``, the criteria's own sub-models, …) — that would silently
-  expand the commitment to dozens of models nobody signed up for. Enumerating the
-  union (not recursing) is what makes "the next criterion-semantics change cannot
-  ship undocumented" mechanical: a new criterion, or a new field on one, fails
-  until it is documented or exempted.
+* **Explicit registry, no recursion.** Only the four registered models are
+  checked; nested models (``AgentConfig``, ``SandboxConfig``, criteria, …) are NOT
+  walked. Walking them would silently expand the documentation commitment to
+  dozens of models nobody signed up for.
 * **Inline-code match, deliberately simple.** A field counts as documented when
   its bare name appears wrapped in Markdown inline-code backticks anywhere in the
   doc. This is a floor, not a proof — a field name that appears in an unrelated
@@ -37,62 +31,21 @@ class impossible to reintroduce: for a small, explicit registry of user-facing
 
 from __future__ import annotations
 
-import ast
 from pathlib import Path
 
 from pydantic import BaseModel
 
 from coder_eval.models import Dataset, RunLimits, SimulationConfig, TaskDefinition
-from coder_eval.models import criteria as _criteria_module
-
-
-_GUIDE = "docs/TASK_DEFINITION_GUIDE.md"
-
-
-def _union_literal_criterion_names() -> list[str]:
-    """Class names in the ``SuccessCriterion = Annotated[A | B | ...]`` literal.
-
-    Read STATICALLY from ``criteria.py`` source, not from the runtime union
-    object. That is the whole point: a plugin's ``coder_eval.plugins`` hook can
-    inject its OWN criterion into the runtime union (and even into the module
-    namespace) at load time — the ``uipath`` SDK adds a ``CliCalledCriterion`` —
-    and such a criterion is documented in its plugin's own repo, not this guide.
-    A plugin cannot edit this repo's source, so the source literal is the
-    authoritative, contamination-proof list of the criteria THIS repo ships.
-    """
-    tree = ast.parse(Path(_criteria_module.__file__).read_text(encoding="utf-8"))
-    for node in tree.body:
-        if isinstance(node, ast.Assign) and any(
-            isinstance(t, ast.Name) and t.id == "SuccessCriterion" for t in node.targets
-        ):
-            # value is ``Annotated[<Name | Name | ...>, Field(...)]``; every
-            # criterion class name ends in "Criterion" (Annotated/Field/discriminator
-            # string do not), so this selects exactly the union members.
-            return [n.id for n in ast.walk(node.value) if isinstance(n, ast.Name) and n.id.endswith("Criterion")]
-    raise AssertionError("SuccessCriterion union assignment not found in criteria.py")
-
-
-def _criterion_models() -> list[type[BaseModel]]:
-    """The in-tree ``SuccessCriterion`` member models, resolved from the source union.
-
-    Names come from the source literal (see :func:`_union_literal_criterion_names`),
-    then each is resolved to its class via ``getattr`` on the criteria module. New
-    in-tree criteria are covered automatically (they're added to the union literal);
-    plugin-injected criteria are never named in the source, so they are excluded
-    even though they exist as runtime attributes.
-    """
-    return [getattr(_criteria_module, name) for name in _union_literal_criterion_names()]
 
 
 # Models the project commits to documenting, paired with the doc page that owns
-# their field reference. The four top-level models are explicit; the criterion
-# union members are enumerated so a new criterion (or field) can't ship undocumented.
+# their field reference. Keep this list SHORT and explicit — every entry is a
+# standing documentation obligation.
 DOCUMENTED_MODELS: list[tuple[type[BaseModel], str]] = [
-    (TaskDefinition, _GUIDE),
-    (RunLimits, _GUIDE),
-    (Dataset, _GUIDE),
-    (SimulationConfig, _GUIDE),
-    *[(m, _GUIDE) for m in _criterion_models()],
+    (TaskDefinition, "docs/TASK_DEFINITION_GUIDE.md"),
+    (RunLimits, "docs/TASK_DEFINITION_GUIDE.md"),
+    (Dataset, "docs/TASK_DEFINITION_GUIDE.md"),
+    (SimulationConfig, "docs/TASK_DEFINITION_GUIDE.md"),
 ]
 
 # Fields deliberately absent from the user docs, with the reason each is not
diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py
index bba7ba31..a1f4da70 100644
--- a/tests/test_custom_lint.py
+++ b/tests/test_custom_lint.py
@@ -910,56 +910,6 @@ def test_registered_models_are_fully_documented(self):
             + "\n".join(f"  {model}: {', '.join(fields)}" for model, fields in sorted(findings.items()))
         )
 
-    def test_criterion_models_match_the_source_union_literal(self):
-        # The enumerated criteria are exactly the class names in the source
-        # ``SuccessCriterion`` union literal, resolved to real module attributes.
-        from coder_eval.models import criteria as crit
-        from tests.lint.doc_schema_parity import _criterion_models, _union_literal_criterion_names
-
-        names = _union_literal_criterion_names()
-        assert names, "expected the source union literal to name criteria"
-        models = _criterion_models()
-        assert [m.__name__ for m in models] == names
-        assert all(getattr(crit, m.__name__, None) is m for m in models)
-
-    def test_plugin_injected_criterion_is_excluded(self, monkeypatch):
-        """A criterion a plugin injects at load time is NOT held to this repo's
-        doc obligation — even when it is a real module attribute, spoofs
-        ``__module__`` to look in-tree, AND is spliced into the runtime union.
-
-        Regression: the ``uipath`` SDK's ``coder_eval.plugins`` hook adds a
-        ``CliCalledCriterion`` (fields ``log``/``positional``) in CI, integrated
-        so thoroughly that neither a ``__module__`` filter nor a module-attribute
-        scan excluded it. The parity list is read from the SOURCE union literal,
-        which a plugin cannot edit, so runtime injection has no effect. This
-        reproduces the full shape and pins that.
-        """
-        from typing import Annotated, Literal
-
-        from pydantic import Field, create_model
-
-        from coder_eval.models import criteria as crit
-        from tests.lint.doc_schema_parity import _criterion_models
-
-        injected = create_model(
-            "CliCalledCriterionStub",
-            __base__=crit.BaseSuccessCriterion,
-            __module__=crit.__name__,  # spoof: looks in-tree
-            type=(Literal["_cli_called_stub"], "_cli_called_stub"),
-            undocumented_plugin_field=(str, ""),
-        )
-        # Integrate it exactly as CI's plugin does: real module attribute + union splice.
-        monkeypatch.setattr(crit, "CliCalledCriterionStub", injected, raising=False)
-        monkeypatch.setattr(
-            crit,
-            "SuccessCriterion",
-            Annotated[crit.FileExistsCriterion | injected, Field(discriminator="type")],
-        )
-
-        names = [m.__name__ for m in _criterion_models()]
-        assert "FileExistsCriterion" in names  # named in the source union literal
-        assert "CliCalledCriterionStub" not in names  # injected at runtime, not in source
-
     def test_exempt_fields_carry_a_reason(self):
         from tests.lint.doc_schema_parity import EXEMPT