Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ Adapter functions are specialized LoRA/aLoRA adapters that add task-specific cap
| `rag` | `rewrite_question(question, context, backend)` | Rewrite question into a retrieval query |
| `rag` | `clarify_query(question, documents, context, backend)` | Generate clarification or return "CLEAR" |
| `rag` | `find_citations(response, documents, context, backend)` | Document sentences supporting the response |
| `rag` | `check_context_relevance(question, document, context, backend)` | Whether a document is relevant; returns a string label (e.g. `'relevant'`, `'partially relevant'`, `'irrelevant'`). **Granite 4.0 only — no adapter for Granite 4.1.** Use `ibm-granite/granite-4.0-micro` as the backend. |
| `rag` | `check_context_relevance(question, document, context, backend)` | **Deprecated.** Granite 4.0 only; no Granite 4.1 adapter and none planned. Will be removed in a future release. Use a prompted relevance check instead. |
| `rag` | `flag_hallucinated_content(response, documents, context, backend)` | Flag potentially hallucinated sentences |

```python
Expand Down
24 changes: 24 additions & 0 deletions docs/docs/advanced/intrinsics.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,30 @@ print(rag.check_answerability(question, docs_not_answerable, context, backend))

## Context relevance

:::warning Deprecated
`check_context_relevance()` is deprecated and will be removed in a future release.
The underlying adapter is Granite 4.0 only and will not receive a Granite 4.1 version.

**There is no direct adapter replacement.** `check_answerability()` addresses a
related but different question — it asks whether a *set* of documents can collectively
answer a question (binary result), while `check_context_relevance()` scores a *single*
document on a three-way scale. They operate at different stages of a RAG pipeline and
are not interchangeable.

For per-document relevance filtering, use a `@generative` function with any current
Granite backend:

```python
from mellea import generative

@generative
def is_relevant(document: str, question: str) -> bool:
"""Determine whether the document contains information relevant to the question."""
```

See [Build a RAG pipeline](../how-to/build-a-rag-pipeline.md) for a full example.
:::

Assess whether a document is relevant to a question:

```python
Expand Down
4 changes: 4 additions & 0 deletions docs/examples/intrinsics/context_relevance.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# pytest: huggingface, e2e

# Deprecated: check_context_relevance() is deprecated and will be removed in a future
# release. The context_relevance adapter is Granite 4.0 only and is not maintained
# for newer models. Use a prompted relevance check with a current Granite backend.

"""Example usage of the context relevance intrinsic for RAG applications.

To run this script from the root of the Mellea source tree, use the command:
Expand Down
2 changes: 2 additions & 0 deletions mellea/backends/adapters/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,8 @@ def effective_capability(self) -> str:
############################################
IntriniscsCatalogEntry(name="answerability", repo_id=_RAG_REPO, revision=_RAG_SHA),
IntriniscsCatalogEntry(name="citations", repo_id=_RAG_REPO, revision=_RAG_SHA),
# Deprecated: Granite 4.0 only; no Granite 4.1 adapter planned.
# Removal tracked alongside check_context_relevance() in rag.py.
IntriniscsCatalogEntry(
name="context_relevance", repo_id=_RAG_REPO, revision=_RAG_SHA
),
Expand Down
15 changes: 15 additions & 0 deletions mellea/stdlib/components/intrinsic/rag.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Adapter functions related to retrieval-augmented generation."""

import collections.abc
import warnings

from ....backends.adapters import AdapterMixin
from ...components import Document
Expand Down Expand Up @@ -162,6 +163,12 @@ def check_context_relevance(
) -> str:
"""Test whether a document is relevant to a user's question.

Deprecated: this function uses a Granite 4.0-only adapter that will not
receive a Granite 4.1 version and is not maintained. There is no direct
adapter replacement — use a @generative function for per-document relevance
filtering. See docs/advanced/intrinsics for a migration example. This
function will be removed in a future release.

Adapter function that checks whether a single document contains part or all of
the answer to a user's question. Does not consider the context in which the
question was asked.
Expand All @@ -181,6 +188,14 @@ def check_context_relevance(
- "irrelevant"
- "partially relevant"
"""
warnings.warn(
"check_context_relevance() is deprecated and will be removed in a future "
"release. The context_relevance adapter is Granite 4.0 only and is not "
"maintained for newer models. Use a @generative function for per-document "
"relevance filtering — see docs/advanced/intrinsics for guidance.",
DeprecationWarning,
stacklevel=2,
)
question, context = _resolve_question(question, context, backend)
document = _coerce_to_document(document)
result_json = call_intrinsic(
Expand Down
3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -378,8 +378,9 @@ filterwarnings = [
"once:.*PydanticSerializationUnexpectedValue.*:UserWarning",


# Keep Watsonx deprecation visible (important for migration)
# Keep specific deprecations visible (important for migration)
"default:.*Watsonx Backend is deprecated.*:DeprecationWarning",
"default:.*check_context_relevance.*:DeprecationWarning",
]

# -----------------------------
Expand Down
18 changes: 3 additions & 15 deletions test/formatters/granite/test_intrinsics_formatters.py
Original file line number Diff line number Diff line change
Expand Up @@ -785,12 +785,9 @@ def test_run_transformers(yaml_json_combo_with_model, gh_run):
responses_str = responses.model_dump_json(indent=4)
print(responses_str[:10000]) # Limit stdout content

# context_relevance_alora: the HF io.yaml is missing max_completion_tokens, so
# transformers falls back to max_length=153 and truncates the JSON mid-string,
# causing JSONDecodeError in result_processor.transform(). Tracked in issue #1301.
if cfg.short_name == "context_relevance_alora":
pytest.xfail(
"context_relevance_alora io.yaml missing max_completion_tokens — see issue #1301"
if cfg.short_name in ("context_relevance_alora", "context_relevance"):
pytest.skip(
"context_relevance adapter deprecated (Granite 4.0 only, not maintained)"
)

# Output processing
Expand Down Expand Up @@ -860,15 +857,6 @@ def test_run_transformers(yaml_json_combo_with_model, gh_run):
)
continue

elif cfg.short_name == "context_relevance":
# context_relevance (non-alora): label is non-deterministic on GPU
# hardware. xfail rather than a bare `continue` so the case
# surfaces in the report instead of silently passing with no
# assertion — see issue #1301 for the keep/remove decision.
pytest.xfail(
"context_relevance label non-deterministic on GPU — see issue #1301"
)

elif cfg.short_name == "context-attribution":
# Context-attribution produces a non-deterministic number and ordering
# of attribution records (HF README explicitly warns about this).
Expand Down
2 changes: 2 additions & 0 deletions test/stdlib/components/intrinsic/test_rag.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ def test_citations(backend):


@pytest.mark.qualitative
@pytest.mark.filterwarnings("ignore::DeprecationWarning")
def test_context_relevance(backend_4_0):
"""Verify that the context relevance intrinsic functions properly."""
context, question, docs = _read_input_json("context_relevance.json")
Expand Down Expand Up @@ -306,6 +307,7 @@ def test_citations_resolve(backend):


@pytest.mark.qualitative
@pytest.mark.filterwarnings("ignore::DeprecationWarning")
def test_context_relevance_resolve(backend_4_0):
"""Verify context relevance when question is resolved from context."""
context, question, docs = _read_input_json("context_relevance.json")
Expand Down
Loading