diff --git a/docs/evals.md b/docs/evals.md index 6a68368..24e05d2 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -371,6 +371,12 @@ Dataclass fields surface as `.` in scores - e.g. `contains_keywords.recall`, `json_valid.valid` (see [Score Namespacing](#score-namespacing)). +`contains_expected` and `word_overlap` require `expected` on the case and +raise when it is `None`: with nothing to compare against, a vacuous pass +(or a fake `overlap=1.0`) would make a case-wiring mistake look like a +healthy run. Attach them only to cases that carry `expected`, per-case if +the case set is mixed. + ## Fixtures Evals use the same fixture system as tests. Expensive setup (database, pipeline, graph) runs once and is shared across all cases. diff --git a/protest/evals/evaluators.py b/protest/evals/evaluators.py index 4e99876..e98e6f4 100644 --- a/protest/evals/evaluators.py +++ b/protest/evals/evaluators.py @@ -67,9 +67,20 @@ def contains_keywords( @evaluator def contains_expected(ctx: EvalContext[Any, str], case_sensitive: bool = False) -> bool: - """Check that the output contains expected_output as a substring.""" + """Check that the output contains expected_output as a substring. + + Requires `expected` on the case: with None it has nothing to check, + and passing vacuously would make a wiring mistake (case data not + reaching the eval) look like a healthy run. + """ if ctx.expected_output is None: - return True + raise ValueError( + f"contains_expected on case '{ctx.name}': expected_output is None. " + f"This evaluator needs EvalCase(expected=...) to have something to " + f"check; a vacuous pass would hide a case-wiring mistake. Set " + f"expected on the case, or attach this evaluator per-case via " + f"EvalCase(evaluators=[...]) when only some cases carry expected." + ) if case_sensitive: return ctx.expected_output in ctx.output return ctx.expected_output.lower() in ctx.output.lower() @@ -147,9 +158,21 @@ def json_valid( @evaluator def word_overlap(ctx: EvalContext[Any, str]) -> WordOverlapResult: - """Compute word overlap between output and expected_output (tracking-only).""" + """Compute word overlap between output and expected_output (tracking-only). + + Requires `expected` on the case: with None there is nothing to overlap + with, and reporting a perfect 1.0 would poison the tracked metric while + hiding a case-wiring mistake. + """ if ctx.expected_output is None: - return WordOverlapResult(overlap=1.0) + raise ValueError( + f"word_overlap on case '{ctx.name}': expected_output is None. " + f"This evaluator needs EvalCase(expected=...) to have something " + f"to compare against; a fake 1.0 would poison the tracked " + f"metric. Set expected on the case, or attach this evaluator " + f"per-case via EvalCase(evaluators=[...]) when only some cases " + f"carry expected." + ) expected = str(ctx.expected_output) expected_words = set(expected.lower().split()) output_words = set(ctx.output.lower().split()) diff --git a/tests/evals/test_e2e.py b/tests/evals/test_e2e.py index 1cc7159..a1db08f 100644 --- a/tests/evals/test_e2e.py +++ b/tests/evals/test_e2e.py @@ -761,6 +761,14 @@ def test_contains_expected(self) -> None: assert e.run(self._make_ctx("Hello World", "world")) is True assert e.run(self._make_ctx("Hello", "world")) is False + def test_contains_expected_none_expected_raises(self) -> None: + """Regression (#118): None expected used to pass vacuously, hiding + case-wiring mistakes.""" + with pytest.raises( + ValueError, match=r"contains_expected .* expected_output is None" + ): + contains_expected.run(self._make_ctx("Hello")) + def test_does_not_contain(self) -> None: e = does_not_contain(forbidden=["cat", "dog"]) assert e.run(self._make_ctx("Yorkshire")).ok is True @@ -839,6 +847,13 @@ def test_word_overlap(self) -> None: assert e.run(self._make_ctx("hello there", "hello world")).overlap == 0.5 assert e.run(self._make_ctx("foo", "hello world")).overlap == 0.0 + def test_word_overlap_none_expected_raises(self) -> None: + """Regression (#118): None expected used to report a fake 1.0.""" + with pytest.raises( + ValueError, match=r"word_overlap .* expected_output is None" + ): + word_overlap.run(self._make_ctx("Hello")) + # --------------------------------------------------------------------------- # Scoring v2: bool verdict, tracking-only metrics