From dc3419604362fed47b158b4e03df532e15422e91 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Wed, 24 Jun 2026 13:30:56 +0100 Subject: [PATCH] =?UTF-8?q?fix(intrinsics):=20deprecate=20check=5Fcontext?= =?UTF-8?q?=5Frelevance=20=E2=80=94=20Granite=204.0=20only=20adapter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The context_relevance adapter is not available for Granite 4.1 or later models and will not be updated (confirmed by @jakelorocco in issue #1301). Changes: - Emit DeprecationWarning in check_context_relevance() with stacklevel=2 - Add deprecation notice to the function docstring - Comment the catalog entry as Granite 4.0 only, not maintained - Replace xfail blocks for context_relevance / context_relevance_alora in test_intrinsics_formatters.py with a single pytest.skip (the aLoRA JSONDecodeError bug is moot once the adapter is deprecated) - Suppress DeprecationWarning in the two qualitative tests that still exercise the function during the deprecation window - Add :::warning Deprecated::: admonition to docs/advanced/intrinsics.md - Add deprecation comment to docs/examples/intrinsics/context_relevance.py - Update AGENTS.md adapter function table to mark as deprecated Full removal (function, catalog entry, tests, example, docs section) is tracked in a follow-up issue to be filed alongside this PR. Closes #1301 Assisted-by: Claude Code Signed-off-by: Nigel Jones --- AGENTS.md | 2 +- docs/docs/advanced/intrinsics.md | 24 +++++++++++++++++++ docs/examples/intrinsics/context_relevance.py | 4 ++++ mellea/backends/adapters/catalog.py | 2 ++ mellea/stdlib/components/intrinsic/rag.py | 15 ++++++++++++ pyproject.toml | 3 ++- .../granite/test_intrinsics_formatters.py | 18 +++----------- test/stdlib/components/intrinsic/test_rag.py | 2 ++ 8 files changed, 53 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ee2b5f31c..dbb0f1692 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -191,7 +191,7 @@ Adapter functions are specialized LoRA/aLoRA adapters that add task-specific cap | `rag` | `rewrite_question(question, context, backend)` | Rewrite question into a retrieval query | | `rag` | `clarify_query(question, documents, context, backend)` | Generate clarification or return "CLEAR" | | `rag` | `find_citations(response, documents, context, backend)` | Document sentences supporting the response | -| `rag` | `check_context_relevance(question, document, context, backend)` | Whether a document is relevant; returns a string label (e.g. `'relevant'`, `'partially relevant'`, `'irrelevant'`). **Granite 4.0 only — no adapter for Granite 4.1.** Use `ibm-granite/granite-4.0-micro` as the backend. | +| `rag` | `check_context_relevance(question, document, context, backend)` | **Deprecated.** Granite 4.0 only; no Granite 4.1 adapter and none planned. Will be removed in a future release. Use a prompted relevance check instead. | | `rag` | `flag_hallucinated_content(response, documents, context, backend)` | Flag potentially hallucinated sentences | ```python diff --git a/docs/docs/advanced/intrinsics.md b/docs/docs/advanced/intrinsics.md index 721985855..cc4892b30 100644 --- a/docs/docs/advanced/intrinsics.md +++ b/docs/docs/advanced/intrinsics.md @@ -74,6 +74,30 @@ print(rag.check_answerability(question, docs_not_answerable, context, backend)) ## Context relevance +:::warning Deprecated +`check_context_relevance()` is deprecated and will be removed in a future release. +The underlying adapter is Granite 4.0 only and will not receive a Granite 4.1 version. + +**There is no direct adapter replacement.** `check_answerability()` addresses a +related but different question — it asks whether a *set* of documents can collectively +answer a question (binary result), while `check_context_relevance()` scores a *single* +document on a three-way scale. They operate at different stages of a RAG pipeline and +are not interchangeable. + +For per-document relevance filtering, use a `@generative` function with any current +Granite backend: + +```python +from mellea import generative + +@generative +def is_relevant(document: str, question: str) -> bool: + """Determine whether the document contains information relevant to the question.""" +``` + +See [Build a RAG pipeline](../how-to/build-a-rag-pipeline.md) for a full example. +::: + Assess whether a document is relevant to a question: ```python diff --git a/docs/examples/intrinsics/context_relevance.py b/docs/examples/intrinsics/context_relevance.py index bd6bfd7d2..90cfc9f23 100644 --- a/docs/examples/intrinsics/context_relevance.py +++ b/docs/examples/intrinsics/context_relevance.py @@ -1,5 +1,9 @@ # pytest: huggingface, e2e +# Deprecated: check_context_relevance() is deprecated and will be removed in a future +# release. The context_relevance adapter is Granite 4.0 only and is not maintained +# for newer models. Use a prompted relevance check with a current Granite backend. + """Example usage of the context relevance intrinsic for RAG applications. To run this script from the root of the Mellea source tree, use the command: diff --git a/mellea/backends/adapters/catalog.py b/mellea/backends/adapters/catalog.py index f45599a36..8d3284ea6 100644 --- a/mellea/backends/adapters/catalog.py +++ b/mellea/backends/adapters/catalog.py @@ -181,6 +181,8 @@ def effective_capability(self) -> str: ############################################ IntriniscsCatalogEntry(name="answerability", repo_id=_RAG_REPO, revision=_RAG_SHA), IntriniscsCatalogEntry(name="citations", repo_id=_RAG_REPO, revision=_RAG_SHA), + # Deprecated: Granite 4.0 only; no Granite 4.1 adapter planned. + # Removal tracked alongside check_context_relevance() in rag.py. IntriniscsCatalogEntry( name="context_relevance", repo_id=_RAG_REPO, revision=_RAG_SHA ), diff --git a/mellea/stdlib/components/intrinsic/rag.py b/mellea/stdlib/components/intrinsic/rag.py index 8bc311d4f..e0aec18dc 100644 --- a/mellea/stdlib/components/intrinsic/rag.py +++ b/mellea/stdlib/components/intrinsic/rag.py @@ -1,6 +1,7 @@ """Adapter functions related to retrieval-augmented generation.""" import collections.abc +import warnings from ....backends.adapters import AdapterMixin from ...components import Document @@ -162,6 +163,12 @@ def check_context_relevance( ) -> str: """Test whether a document is relevant to a user's question. + Deprecated: this function uses a Granite 4.0-only adapter that will not + receive a Granite 4.1 version and is not maintained. There is no direct + adapter replacement — use a @generative function for per-document relevance + filtering. See docs/advanced/intrinsics for a migration example. This + function will be removed in a future release. + Adapter function that checks whether a single document contains part or all of the answer to a user's question. Does not consider the context in which the question was asked. @@ -181,6 +188,14 @@ def check_context_relevance( - "irrelevant" - "partially relevant" """ + warnings.warn( + "check_context_relevance() is deprecated and will be removed in a future " + "release. The context_relevance adapter is Granite 4.0 only and is not " + "maintained for newer models. Use a @generative function for per-document " + "relevance filtering — see docs/advanced/intrinsics for guidance.", + DeprecationWarning, + stacklevel=2, + ) question, context = _resolve_question(question, context, backend) document = _coerce_to_document(document) result_json = call_intrinsic( diff --git a/pyproject.toml b/pyproject.toml index afe620431..08244f104 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -378,8 +378,9 @@ filterwarnings = [ "once:.*PydanticSerializationUnexpectedValue.*:UserWarning", - # Keep Watsonx deprecation visible (important for migration) + # Keep specific deprecations visible (important for migration) "default:.*Watsonx Backend is deprecated.*:DeprecationWarning", + "default:.*check_context_relevance.*:DeprecationWarning", ] # ----------------------------- diff --git a/test/formatters/granite/test_intrinsics_formatters.py b/test/formatters/granite/test_intrinsics_formatters.py index 3df522ca8..c03a6f9e2 100644 --- a/test/formatters/granite/test_intrinsics_formatters.py +++ b/test/formatters/granite/test_intrinsics_formatters.py @@ -785,12 +785,9 @@ def test_run_transformers(yaml_json_combo_with_model, gh_run): responses_str = responses.model_dump_json(indent=4) print(responses_str[:10000]) # Limit stdout content - # context_relevance_alora: the HF io.yaml is missing max_completion_tokens, so - # transformers falls back to max_length=153 and truncates the JSON mid-string, - # causing JSONDecodeError in result_processor.transform(). Tracked in issue #1301. - if cfg.short_name == "context_relevance_alora": - pytest.xfail( - "context_relevance_alora io.yaml missing max_completion_tokens — see issue #1301" + if cfg.short_name in ("context_relevance_alora", "context_relevance"): + pytest.skip( + "context_relevance adapter deprecated (Granite 4.0 only, not maintained)" ) # Output processing @@ -860,15 +857,6 @@ def test_run_transformers(yaml_json_combo_with_model, gh_run): ) continue - elif cfg.short_name == "context_relevance": - # context_relevance (non-alora): label is non-deterministic on GPU - # hardware. xfail rather than a bare `continue` so the case - # surfaces in the report instead of silently passing with no - # assertion — see issue #1301 for the keep/remove decision. - pytest.xfail( - "context_relevance label non-deterministic on GPU — see issue #1301" - ) - elif cfg.short_name == "context-attribution": # Context-attribution produces a non-deterministic number and ordering # of attribution records (HF README explicitly warns about this). diff --git a/test/stdlib/components/intrinsic/test_rag.py b/test/stdlib/components/intrinsic/test_rag.py index f0a85f2ce..485f3411f 100644 --- a/test/stdlib/components/intrinsic/test_rag.py +++ b/test/stdlib/components/intrinsic/test_rag.py @@ -174,6 +174,7 @@ def test_citations(backend): @pytest.mark.qualitative +@pytest.mark.filterwarnings("ignore::DeprecationWarning") def test_context_relevance(backend_4_0): """Verify that the context relevance intrinsic functions properly.""" context, question, docs = _read_input_json("context_relevance.json") @@ -306,6 +307,7 @@ def test_citations_resolve(backend): @pytest.mark.qualitative +@pytest.mark.filterwarnings("ignore::DeprecationWarning") def test_context_relevance_resolve(backend_4_0): """Verify context relevance when question is resolved from context.""" context, question, docs = _read_input_json("context_relevance.json")