evals: contains_expected / word_overlap raise on empty or whitespace expected

docs/evals.md

@@ -371,11 +371,11 @@ Dataclass fields surface as `<evaluator>.<field>` in scores - e.g.
 `contains_keywords.recall`, `json_valid.valid` (see
 [Score Namespacing](#score-namespacing)).
 
-`contains_expected` and `word_overlap` require `expected` on the case and
-raise when it is `None`: with nothing to compare against, a vacuous pass
-(or a fake `overlap=1.0`) would make a case-wiring mistake look like a
-healthy run. Attach them only to cases that carry `expected`, per-case if
-the case set is mixed.
+`contains_expected` and `word_overlap` require a **non-empty** `expected` on
+the case and raise when it is `None`, empty, or whitespace-only: with nothing
+to compare against, a vacuous pass (`"" in output` is always true) or a fake
+`overlap=1.0` would make a case-wiring mistake look like a healthy run. Attach
+them only to cases that carry `expected`, per-case if the case set is mixed.
 
 ## Fixtures
 

protest/evals/evaluators.py

@@ -69,9 +69,10 @@ def contains_keywords(
 def contains_expected(ctx: EvalContext[Any, str], case_sensitive: bool = False) -> bool:
     """Check that the output contains expected_output as a substring.
 
-    Requires `expected` on the case: with None it has nothing to check,
-    and passing vacuously would make a wiring mistake (case data not
-    reaching the eval) look like a healthy run.
+    Requires a non-empty `expected` on the case: with None there is nothing
+    to check, and with an empty or whitespace-only string ``"" in output`` is
+    true for every output. Either way a vacuous pass would make a wiring
+    mistake (case data not reaching the eval) look like a healthy run.
     """
     if ctx.expected_output is None:
         raise ValueError(
@@ -81,6 +82,14 @@ def contains_expected(ctx: EvalContext[Any, str], case_sensitive: bool = False)
             f"expected on the case, or attach this evaluator per-case via "
             f"EvalCase(evaluators=[...]) when only some cases carry expected."
         )
+    if isinstance(ctx.expected_output, str) and not ctx.expected_output.strip():
+        raise ValueError(
+            f"contains_expected on case '{ctx.name}': expected_output is empty "
+            f"or whitespace-only. An empty string is a substring of every "
+            f"output, so the check would pass vacuously regardless of the "
+            f"result. Set a non-empty expected on the case, or attach this "
+            f"evaluator per-case via EvalCase(evaluators=[...])."
+        )
     if case_sensitive:
         return ctx.expected_output in ctx.output
     return ctx.expected_output.lower() in ctx.output.lower()
@@ -160,9 +169,10 @@ def json_valid(
 def word_overlap(ctx: EvalContext[Any, str]) -> WordOverlapResult:
     """Compute word overlap between output and expected_output (tracking-only).
 
-    Requires `expected` on the case: with None there is nothing to overlap
-    with, and reporting a perfect 1.0 would poison the tracked metric while
-    hiding a case-wiring mistake.
+    Requires a non-empty `expected` on the case: with None there is nothing
+    to overlap with, and with an empty or whitespace-only string there are no
+    words to compare against. Either way reporting a perfect 1.0 would poison
+    the tracked metric while hiding a case-wiring mistake.
     """
     if ctx.expected_output is None:
         raise ValueError(
@@ -177,7 +187,13 @@ def word_overlap(ctx: EvalContext[Any, str]) -> WordOverlapResult:
     expected_words = set(expected.lower().split())
     output_words = set(ctx.output.lower().split())
     if not expected_words:
-        return WordOverlapResult(overlap=1.0)
+        raise ValueError(
+            f"word_overlap on case '{ctx.name}': expected_output is empty or "
+            f"whitespace-only, so there are no words to compare against. "
+            f"Reporting overlap=1.0 would poison the tracked metric. Set a "
+            f"non-empty expected on the case, or attach this evaluator "
+            f"per-case via EvalCase(evaluators=[...])."
+        )
     return WordOverlapResult(
         overlap=len(expected_words & output_words) / len(expected_words),
     )

tests/evals/test_e2e.py

@@ -769,6 +769,19 @@ def test_contains_expected_none_expected_raises(self) -> None:
         ):
             contains_expected.run(self._make_ctx("Hello"))
 
+    def test_contains_expected_empty_expected_raises(self) -> None:
+        """#126: '' is a substring of every output -> vacuous pass."""
+        with pytest.raises(
+            ValueError, match=r"contains_expected .* empty or whitespace-only"
+        ):
+            contains_expected.run(self._make_ctx("Hello", ""))
+
+    def test_contains_expected_whitespace_expected_raises(self) -> None:
+        with pytest.raises(
+            ValueError, match=r"contains_expected .* empty or whitespace-only"
+        ):
+            contains_expected.run(self._make_ctx("Hello", "   "))
+
     def test_does_not_contain(self) -> None:
         e = does_not_contain(forbidden=["cat", "dog"])
         assert e.run(self._make_ctx("Yorkshire")).ok is True
@@ -854,6 +867,19 @@ def test_word_overlap_none_expected_raises(self) -> None:
         ):
             word_overlap.run(self._make_ctx("Hello"))
 
+    def test_word_overlap_empty_expected_raises(self) -> None:
+        """#126: empty expected -> no words -> fabricated overlap=1.0."""
+        with pytest.raises(
+            ValueError, match=r"word_overlap .* empty or whitespace-only"
+        ):
+            word_overlap.run(self._make_ctx("Hello", ""))
+
+    def test_word_overlap_whitespace_expected_raises(self) -> None:
+        with pytest.raises(
+            ValueError, match=r"word_overlap .* empty or whitespace-only"
+        ):
+            word_overlap.run(self._make_ctx("Hello", "   "))
+
 
 # ---------------------------------------------------------------------------
 # Scoring v2: bool verdict, tracking-only metrics