diff --git a/docs/evals.md b/docs/evals.md index 24e05d2..f69f89b 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -371,11 +371,11 @@ Dataclass fields surface as `.` in scores - e.g. `contains_keywords.recall`, `json_valid.valid` (see [Score Namespacing](#score-namespacing)). -`contains_expected` and `word_overlap` require `expected` on the case and -raise when it is `None`: with nothing to compare against, a vacuous pass -(or a fake `overlap=1.0`) would make a case-wiring mistake look like a -healthy run. Attach them only to cases that carry `expected`, per-case if -the case set is mixed. +`contains_expected` and `word_overlap` require a **non-empty** `expected` on +the case and raise when it is `None`, empty, or whitespace-only: with nothing +to compare against, a vacuous pass (`"" in output` is always true) or a fake +`overlap=1.0` would make a case-wiring mistake look like a healthy run. Attach +them only to cases that carry `expected`, per-case if the case set is mixed. ## Fixtures diff --git a/protest/evals/evaluators.py b/protest/evals/evaluators.py index e98e6f4..057f3f2 100644 --- a/protest/evals/evaluators.py +++ b/protest/evals/evaluators.py @@ -69,9 +69,10 @@ def contains_keywords( def contains_expected(ctx: EvalContext[Any, str], case_sensitive: bool = False) -> bool: """Check that the output contains expected_output as a substring. - Requires `expected` on the case: with None it has nothing to check, - and passing vacuously would make a wiring mistake (case data not - reaching the eval) look like a healthy run. + Requires a non-empty `expected` on the case: with None there is nothing + to check, and with an empty or whitespace-only string ``"" in output`` is + true for every output. Either way a vacuous pass would make a wiring + mistake (case data not reaching the eval) look like a healthy run. """ if ctx.expected_output is None: raise ValueError( @@ -81,6 +82,14 @@ def contains_expected(ctx: EvalContext[Any, str], case_sensitive: bool = False) f"expected on the case, or attach this evaluator per-case via " f"EvalCase(evaluators=[...]) when only some cases carry expected." ) + if isinstance(ctx.expected_output, str) and not ctx.expected_output.strip(): + raise ValueError( + f"contains_expected on case '{ctx.name}': expected_output is empty " + f"or whitespace-only. An empty string is a substring of every " + f"output, so the check would pass vacuously regardless of the " + f"result. Set a non-empty expected on the case, or attach this " + f"evaluator per-case via EvalCase(evaluators=[...])." + ) if case_sensitive: return ctx.expected_output in ctx.output return ctx.expected_output.lower() in ctx.output.lower() @@ -160,9 +169,10 @@ def json_valid( def word_overlap(ctx: EvalContext[Any, str]) -> WordOverlapResult: """Compute word overlap between output and expected_output (tracking-only). - Requires `expected` on the case: with None there is nothing to overlap - with, and reporting a perfect 1.0 would poison the tracked metric while - hiding a case-wiring mistake. + Requires a non-empty `expected` on the case: with None there is nothing + to overlap with, and with an empty or whitespace-only string there are no + words to compare against. Either way reporting a perfect 1.0 would poison + the tracked metric while hiding a case-wiring mistake. """ if ctx.expected_output is None: raise ValueError( @@ -177,7 +187,13 @@ def word_overlap(ctx: EvalContext[Any, str]) -> WordOverlapResult: expected_words = set(expected.lower().split()) output_words = set(ctx.output.lower().split()) if not expected_words: - return WordOverlapResult(overlap=1.0) + raise ValueError( + f"word_overlap on case '{ctx.name}': expected_output is empty or " + f"whitespace-only, so there are no words to compare against. " + f"Reporting overlap=1.0 would poison the tracked metric. Set a " + f"non-empty expected on the case, or attach this evaluator " + f"per-case via EvalCase(evaluators=[...])." + ) return WordOverlapResult( overlap=len(expected_words & output_words) / len(expected_words), ) diff --git a/tests/evals/test_e2e.py b/tests/evals/test_e2e.py index a1db08f..06e2d1d 100644 --- a/tests/evals/test_e2e.py +++ b/tests/evals/test_e2e.py @@ -769,6 +769,19 @@ def test_contains_expected_none_expected_raises(self) -> None: ): contains_expected.run(self._make_ctx("Hello")) + def test_contains_expected_empty_expected_raises(self) -> None: + """#126: '' is a substring of every output -> vacuous pass.""" + with pytest.raises( + ValueError, match=r"contains_expected .* empty or whitespace-only" + ): + contains_expected.run(self._make_ctx("Hello", "")) + + def test_contains_expected_whitespace_expected_raises(self) -> None: + with pytest.raises( + ValueError, match=r"contains_expected .* empty or whitespace-only" + ): + contains_expected.run(self._make_ctx("Hello", " ")) + def test_does_not_contain(self) -> None: e = does_not_contain(forbidden=["cat", "dog"]) assert e.run(self._make_ctx("Yorkshire")).ok is True @@ -854,6 +867,19 @@ def test_word_overlap_none_expected_raises(self) -> None: ): word_overlap.run(self._make_ctx("Hello")) + def test_word_overlap_empty_expected_raises(self) -> None: + """#126: empty expected -> no words -> fabricated overlap=1.0.""" + with pytest.raises( + ValueError, match=r"word_overlap .* empty or whitespace-only" + ): + word_overlap.run(self._make_ctx("Hello", "")) + + def test_word_overlap_whitespace_expected_raises(self) -> None: + with pytest.raises( + ValueError, match=r"word_overlap .* empty or whitespace-only" + ): + word_overlap.run(self._make_ctx("Hello", " ")) + # --------------------------------------------------------------------------- # Scoring v2: bool verdict, tracking-only metrics