fix(evals): contains_expected and word_overlap raise when expected is None

docs/evals.md

@@ -371,6 +371,12 @@ Dataclass fields surface as `<evaluator>.<field>` in scores - e.g.
 `contains_keywords.recall`, `json_valid.valid` (see
 [Score Namespacing](#score-namespacing)).
 
+`contains_expected` and `word_overlap` require `expected` on the case and
+raise when it is `None`: with nothing to compare against, a vacuous pass
+(or a fake `overlap=1.0`) would make a case-wiring mistake look like a
+healthy run. Attach them only to cases that carry `expected`, per-case if
+the case set is mixed.
+
 ## Fixtures
 
 Evals use the same fixture system as tests. Expensive setup (database, pipeline, graph) runs once and is shared across all cases.

protest/evals/evaluators.py

@@ -67,9 +67,20 @@ def contains_keywords(
 
 @evaluator
 def contains_expected(ctx: EvalContext[Any, str], case_sensitive: bool = False) -> bool:
-    """Check that the output contains expected_output as a substring."""
+    """Check that the output contains expected_output as a substring.
+
+    Requires `expected` on the case: with None it has nothing to check,
+    and passing vacuously would make a wiring mistake (case data not
+    reaching the eval) look like a healthy run.
+    """
     if ctx.expected_output is None:
-        return True
+        raise ValueError(
+            f"contains_expected on case '{ctx.name}': expected_output is None. "
+            f"This evaluator needs EvalCase(expected=...) to have something to "
+            f"check; a vacuous pass would hide a case-wiring mistake. Set "
+            f"expected on the case, or attach this evaluator per-case via "
+            f"EvalCase(evaluators=[...]) when only some cases carry expected."
+        )
     if case_sensitive:
         return ctx.expected_output in ctx.output
     return ctx.expected_output.lower() in ctx.output.lower()
@@ -147,9 +158,21 @@ def json_valid(
 
 @evaluator
 def word_overlap(ctx: EvalContext[Any, str]) -> WordOverlapResult:
-    """Compute word overlap between output and expected_output (tracking-only)."""
+    """Compute word overlap between output and expected_output (tracking-only).
+
+    Requires `expected` on the case: with None there is nothing to overlap
+    with, and reporting a perfect 1.0 would poison the tracked metric while
+    hiding a case-wiring mistake.
+    """
     if ctx.expected_output is None:
-        return WordOverlapResult(overlap=1.0)
+        raise ValueError(
+            f"word_overlap on case '{ctx.name}': expected_output is None. "
+            f"This evaluator needs EvalCase(expected=...) to have something "
+            f"to compare against; a fake 1.0 would poison the tracked "
+            f"metric. Set expected on the case, or attach this evaluator "
+            f"per-case via EvalCase(evaluators=[...]) when only some cases "
+            f"carry expected."
+        )
     expected = str(ctx.expected_output)
     expected_words = set(expected.lower().split())
     output_words = set(ctx.output.lower().split())

tests/evals/test_e2e.py

@@ -761,6 +761,14 @@ def test_contains_expected(self) -> None:
         assert e.run(self._make_ctx("Hello World", "world")) is True
         assert e.run(self._make_ctx("Hello", "world")) is False
 
+    def test_contains_expected_none_expected_raises(self) -> None:
+        """Regression (#118): None expected used to pass vacuously, hiding
+        case-wiring mistakes."""
+        with pytest.raises(
+            ValueError, match=r"contains_expected .* expected_output is None"
+        ):
+            contains_expected.run(self._make_ctx("Hello"))
+
     def test_does_not_contain(self) -> None:
         e = does_not_contain(forbidden=["cat", "dog"])
         assert e.run(self._make_ctx("Yorkshire")).ok is True
@@ -839,6 +847,13 @@ def test_word_overlap(self) -> None:
         assert e.run(self._make_ctx("hello there", "hello world")).overlap == 0.5
         assert e.run(self._make_ctx("foo", "hello world")).overlap == 0.0
 
+    def test_word_overlap_none_expected_raises(self) -> None:
+        """Regression (#118): None expected used to report a fake 1.0."""
+        with pytest.raises(
+            ValueError, match=r"word_overlap .* expected_output is None"
+        ):
+            word_overlap.run(self._make_ctx("Hello"))
+
 
 # ---------------------------------------------------------------------------
 # Scoring v2: bool verdict, tracking-only metrics