From 99442d3d435c65e7effb1a5e58a303bbb4e52212 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Thu, 4 Jun 2026 09:01:15 +0100 Subject: [PATCH 1/4] fix(tests): skip richdocument fixture on HuggingFace network errors Module-scoped rd fixture now skips gracefully on 429 / network failures instead of erroring all dependent tests. Fixes intermittent CI failures reported in #1198. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- test/stdlib/components/docs/test_richdocument.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/stdlib/components/docs/test_richdocument.py b/test/stdlib/components/docs/test_richdocument.py index b6011d7cd..f49755b98 100644 --- a/test/stdlib/components/docs/test_richdocument.py +++ b/test/stdlib/components/docs/test_richdocument.py @@ -20,9 +20,14 @@ def rd() -> RichDocument: # Use a specific document so we can test some of the functionality # related to extracting and transforming text. - return RichDocument.from_document_file( - "https://arxiv.org/pdf/1906.04043", do_ocr=False - ) + try: + return RichDocument.from_document_file( + "https://arxiv.org/pdf/1906.04043", do_ocr=False + ) + except Exception as e: + pytest.skip( + f"HuggingFace Hub or network unavailable — {e}", allow_module_level=True + ) def test_richdocument_basics(rd: RichDocument): From 25116c81becf718af9ee4b9abbcba658da21eadc Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Thu, 4 Jun 2026 10:23:47 +0100 Subject: [PATCH 2/4] fix(tests): skip groundedness backend fixture on HuggingFace network errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Matches the skip pattern from test_richdocument.py — wraps LocalHFBackend construction in try/except so tests skip rather than error when the Hub is unavailable or rate-limiting. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- test/stdlib/requirements/test_groundedness_requirement.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/stdlib/requirements/test_groundedness_requirement.py b/test/stdlib/requirements/test_groundedness_requirement.py index 2d487b96a..f4bdc1973 100644 --- a/test/stdlib/requirements/test_groundedness_requirement.py +++ b/test/stdlib/requirements/test_groundedness_requirement.py @@ -15,7 +15,10 @@ @pytest.fixture def backend(): """Provide HuggingFace backend for tests.""" - return LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") + try: + return LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") + except Exception as e: + pytest.skip(f"HuggingFace Hub or network unavailable — {e}") @pytest.fixture From e5d207f45810b26038f39341d7c6ba2ca0be86a4 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Thu, 4 Jun 2026 11:20:55 +0100 Subject: [PATCH 3/4] test: centralise HF Hub skip guard into hf_skip() context manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds hf_skip() to test/conftest.py: a single contextlib.contextmanager that catches (OSError, requests.exceptions.RequestException) — the two exception families that cover HfHubHTTPError, LocalEntryNotFoundError, and every transformers-wrapped Hub error — and converts them into a clean pytest.skip rather than an ERROR. Applies hf_skip() to all 11 fixtures (across 11 test files) that call LocalHFBackend(), start_session("hf", ...), RichDocument.from_document_file(), or obtain_io_yaml() without a guard. Retrofits the two pre-existing hand-rolled guards in test_intrinsics_formatters.py to use the helper, removing the now-redundant LocalEntryNotFoundError import from that file. This does not address the underlying cause of 429 rate-limit responses or other mitigations such as authenticated Hub access tokens. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- test/backends/test_huggingface.py | 20 ++++++++------- test/backends/test_huggingface_tools.py | 8 +++--- test/conftest.py | 25 +++++++++++++++++++ .../granite/test_intrinsics_formatters.py | 10 +++----- .../components/docs/test_richdocument.py | 7 ++---- test/stdlib/components/intrinsic/test_core.py | 5 ++-- .../components/intrinsic/test_guardian.py | 5 ++-- test/stdlib/components/intrinsic/test_rag.py | 7 ++++-- .../test_groundedness_requirement.py | 5 ++-- .../test_groundedness_requirement_e2e.py | 4 ++- test/stdlib/test_spans.py | 12 +++++---- test/telemetry/test_metrics_backend.py | 10 +++++--- 12 files changed, 75 insertions(+), 43 deletions(-) diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index e6a8d5236..bb9805729 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -47,6 +47,7 @@ from mellea.stdlib.components import Intrinsic, Message from mellea.stdlib.context import ChatContext, SimpleContext from mellea.stdlib.requirements import ALoraRequirement, LLMaJRequirement +from test.conftest import hf_skip @pytest.fixture(scope="module") @@ -59,15 +60,16 @@ def backend(): against this granite-3.3 backend will fail once revision pinning is wired through in phase-2.2 (#1141). Other intrinsics are not affected. """ - backend = LocalHFBackend( - model_id=model_ids.IBM_GRANITE_4_1_3B, cache=SimpleLRUCache(5) - ) - backend.add_adapter( - IntrinsicAdapter("requirement-check", base_model_name=backend.base_model_name) - ) - backend.add_adapter( - IntrinsicAdapter("answerability", base_model_name=backend.base_model_name) - ) + with hf_skip(): + backend = LocalHFBackend( + model_id=model_ids.IBM_GRANITE_4_1_3B, cache=SimpleLRUCache(5) + ) + backend.add_adapter( + IntrinsicAdapter("requirement-check", base_model_name=backend.base_model_name) + ) + backend.add_adapter( + IntrinsicAdapter("answerability", base_model_name=backend.base_model_name) + ) yield backend from test.conftest import cleanup_gpu_backend diff --git a/test/backends/test_huggingface_tools.py b/test/backends/test_huggingface_tools.py index b2a72176f..aa658ac1f 100644 --- a/test/backends/test_huggingface_tools.py +++ b/test/backends/test_huggingface_tools.py @@ -25,14 +25,16 @@ from mellea.backends.cache import SimpleLRUCache from mellea.backends.huggingface import LocalHFBackend from mellea.stdlib.context import ChatContext +from test.conftest import hf_skip @pytest.fixture(scope="module") def backend(): """Shared HuggingFace backend for all tests in this module.""" - backend = LocalHFBackend( - model_id=model_ids.MISTRALAI_MISTRAL_0_3_7B, cache=SimpleLRUCache(5) - ) + with hf_skip(): + backend = LocalHFBackend( + model_id=model_ids.MISTRALAI_MISTRAL_0_3_7B, cache=SimpleLRUCache(5) + ) # add_granite_aloras(backend) yield backend diff --git a/test/conftest.py b/test/conftest.py index cf6fed7ae..d424d5e15 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,3 +1,4 @@ +import contextlib import gc import os import subprocess @@ -9,6 +10,30 @@ from mellea.core import MelleaLogger +# ============================================================================ +# HuggingFace Hub Skip Helper +# ============================================================================ + + +@contextlib.contextmanager +def hf_skip(): + """Skip the current test/fixture when a HuggingFace Hub call fails. + + Catches OSError (covers HfHubHTTPError, LocalEntryNotFoundError, and + transformers-wrapped Hub errors) plus requests network errors. Use this + in every fixture that downloads model weights or YAML files from the Hub + so a 429 or transient network blip results in a clean skip rather than an + ERROR that masks unrelated failures. + """ + try: + yield + except (OSError, requests.exceptions.RequestException) as e: + pytest.skip( + f"HuggingFace Hub not accessible: {type(e).__name__}: {e}", + allow_module_level=True, + ) + + # Try to import optional dependencies for system detection try: import psutil diff --git a/test/formatters/granite/test_intrinsics_formatters.py b/test/formatters/granite/test_intrinsics_formatters.py index 2231255a6..e52d49509 100644 --- a/test/formatters/granite/test_intrinsics_formatters.py +++ b/test/formatters/granite/test_intrinsics_formatters.py @@ -21,7 +21,6 @@ torch = pytest.importorskip("torch", reason="torch not installed — install mellea[hf]") import yaml -from huggingface_hub.errors import LocalEntryNotFoundError # First Party from mellea.formatters.granite import ( @@ -32,6 +31,7 @@ ) from mellea.formatters.granite.base import util as base_util from mellea.formatters.granite.intrinsics import json_util, util as intrinsics_util +from test.conftest import hf_skip def _read_file(name): @@ -147,15 +147,13 @@ def _resolve_yaml(self): object. Called at fixture creation (execution time) to prevent collection time errors. """ if not self.yaml_file: - try: + with hf_skip(): self.yaml_file = intrinsics_util.obtain_io_yaml( self.task, self.base_model_id, self.repo_id, revision=self.revision, # type: ignore ) - except (LocalEntryNotFoundError, requests.exceptions.RequestException) as e: - pytest.skip(f"HuggingFace Hub not accessible: {type(e).__name__}: {e}") return self @@ -537,12 +535,10 @@ def test_read_yaml(): IntrinsicsRewriter(config_file=_INPUT_YAML_DIR / "answerability.yaml") # Read from Hugging Face hub. - try: + with hf_skip(): local_path = intrinsics_util.obtain_io_yaml( "answerability", "granite-4.0-micro", _RAG_INTRINSICS_REPO_NAME ) - except (LocalEntryNotFoundError, requests.exceptions.RequestException) as e: - pytest.skip(f"HuggingFace Hub not accessible: {type(e).__name__}: {e}") IntrinsicsRewriter(config_file=local_path) diff --git a/test/stdlib/components/docs/test_richdocument.py b/test/stdlib/components/docs/test_richdocument.py index f49755b98..06e861a4d 100644 --- a/test/stdlib/components/docs/test_richdocument.py +++ b/test/stdlib/components/docs/test_richdocument.py @@ -11,6 +11,7 @@ import mellea from mellea.core import TemplateRepresentation from mellea.stdlib.components.docs.richdocument import RichDocument, Table +from test.conftest import hf_skip from test.predicates import require_gpu pytestmark = pytest.mark.integration @@ -20,14 +21,10 @@ def rd() -> RichDocument: # Use a specific document so we can test some of the functionality # related to extracting and transforming text. - try: + with hf_skip(): return RichDocument.from_document_file( "https://arxiv.org/pdf/1906.04043", do_ocr=False ) - except Exception as e: - pytest.skip( - f"HuggingFace Hub or network unavailable — {e}", allow_module_level=True - ) def test_richdocument_basics(rd: RichDocument): diff --git a/test/stdlib/components/intrinsic/test_core.py b/test/stdlib/components/intrinsic/test_core.py index 3c8df0eb9..b6724512d 100644 --- a/test/stdlib/components/intrinsic/test_core.py +++ b/test/stdlib/components/intrinsic/test_core.py @@ -17,7 +17,7 @@ from mellea.stdlib.components.intrinsic import core from mellea.stdlib.components.intrinsic._util import call_intrinsic from mellea.stdlib.context import ChatContext -from test.conftest import cleanup_gpu_backend +from test.conftest import cleanup_gpu_backend, hf_skip from test.predicates import require_gpu from test.stdlib.components.intrinsic.test_rag import ( _read_input_json as _read_rag_input_json, @@ -48,7 +48,8 @@ def _backend(): # Prevent thrashing if the default device is CPU torch.set_num_threads(4) - backend_ = LocalHFBackend(model_id=BASE_MODEL) + with hf_skip(): + backend_ = LocalHFBackend(model_id=BASE_MODEL) yield backend_ # Code after yield is cleanup code. diff --git a/test/stdlib/components/intrinsic/test_guardian.py b/test/stdlib/components/intrinsic/test_guardian.py index 04d3e9214..3e3663be4 100644 --- a/test/stdlib/components/intrinsic/test_guardian.py +++ b/test/stdlib/components/intrinsic/test_guardian.py @@ -14,7 +14,7 @@ from mellea.stdlib.components import Document, Message from mellea.stdlib.components.intrinsic import guardian from mellea.stdlib.context import ChatContext -from test.conftest import cleanup_gpu_backend +from test.conftest import cleanup_gpu_backend, hf_skip from test.predicates import require_gpu # Skip entire module in CI since all tests are qualitative @@ -37,7 +37,8 @@ def _backend(): """Backend used by the tests in this file. Module-scoped to avoid reloading the model for each test.""" torch.set_num_threads(4) - backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_MICRO_3B.hf_model_name) # type: ignore + with hf_skip(): + backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_MICRO_3B.hf_model_name) # type: ignore yield backend_ cleanup_gpu_backend(backend_, "test_guardian") diff --git a/test/stdlib/components/intrinsic/test_rag.py b/test/stdlib/components/intrinsic/test_rag.py index 3bfc591bf..f0a85f2ce 100644 --- a/test/stdlib/components/intrinsic/test_rag.py +++ b/test/stdlib/components/intrinsic/test_rag.py @@ -15,6 +15,7 @@ from mellea.stdlib.components import Document, Message from mellea.stdlib.components.intrinsic import rag from mellea.stdlib.context import ChatContext +from test.conftest import hf_skip from test.predicates import require_gpu # Skip entire module in CI since all 7 tests are qualitative @@ -42,7 +43,8 @@ def _backend(): torch.set_num_threads(4) # No adapters for hybrid version. - backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_1_3B.hf_model_name) + with hf_skip(): + backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_1_3B.hf_model_name) yield backend_ from test.conftest import cleanup_gpu_backend @@ -57,7 +59,8 @@ def _backend_4_0(): torch.set_num_threads(4) # No adapters for hybrid version. - backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_MICRO_3B.hf_model_name) + with hf_skip(): + backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_MICRO_3B.hf_model_name) yield backend_ from test.conftest import cleanup_gpu_backend diff --git a/test/stdlib/requirements/test_groundedness_requirement.py b/test/stdlib/requirements/test_groundedness_requirement.py index f4bdc1973..d9f912e5b 100644 --- a/test/stdlib/requirements/test_groundedness_requirement.py +++ b/test/stdlib/requirements/test_groundedness_requirement.py @@ -9,16 +9,15 @@ from mellea.stdlib.components import Document, Message from mellea.stdlib.context import ChatContext from mellea.stdlib.requirements.rag import GroundednessRequirement +from test.conftest import hf_skip from test.predicates import require_gpu @pytest.fixture def backend(): """Provide HuggingFace backend for tests.""" - try: + with hf_skip(): return LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") - except Exception as e: - pytest.skip(f"HuggingFace Hub or network unavailable — {e}") @pytest.fixture diff --git a/test/stdlib/requirements/test_groundedness_requirement_e2e.py b/test/stdlib/requirements/test_groundedness_requirement_e2e.py index fbbe33e6a..2d4b9a7e1 100644 --- a/test/stdlib/requirements/test_groundedness_requirement_e2e.py +++ b/test/stdlib/requirements/test_groundedness_requirement_e2e.py @@ -6,13 +6,15 @@ from mellea.stdlib.components import Document, Message from mellea.stdlib.context import ChatContext from mellea.stdlib.requirements.rag import GroundednessRequirement +from test.conftest import hf_skip from test.predicates import require_gpu @pytest.fixture(scope="session") def backend(): """Provide HuggingFace backend for tests (session-scoped to avoid OOM).""" - return LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") + with hf_skip(): + return LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") @pytest.fixture diff --git a/test/stdlib/test_spans.py b/test/stdlib/test_spans.py index 7914fa02a..1946025e2 100644 --- a/test/stdlib/test_spans.py +++ b/test/stdlib/test_spans.py @@ -10,6 +10,7 @@ from mellea.core import CBlock from mellea.stdlib.components import SimpleComponent from mellea.stdlib.session import MelleaSession, start_session +from test.conftest import hf_skip from test.predicates import require_gpu # Module-level markers for all tests using Granite 4.1 3B model @@ -19,11 +20,12 @@ # We edit the context type in the async tests below. Don't change the scope here. @pytest.fixture(scope="function") def m_session(gh_run): - m = start_session( - "hf", - model_id=IBM_GRANITE_4_1_3B, - model_options={ModelOption.MAX_NEW_TOKENS: 64}, - ) + with hf_skip(): + m = start_session( + "hf", + model_id=IBM_GRANITE_4_1_3B, + model_options={ModelOption.MAX_NEW_TOKENS: 64}, + ) yield m from test.conftest import cleanup_gpu_backend diff --git a/test/telemetry/test_metrics_backend.py b/test/telemetry/test_metrics_backend.py index c538accd2..2119501cf 100644 --- a/test/telemetry/test_metrics_backend.py +++ b/test/telemetry/test_metrics_backend.py @@ -16,6 +16,7 @@ ) from mellea.stdlib.components import Message from mellea.stdlib.context import SimpleContext +from test.conftest import hf_skip from test.predicates import require_api_key, require_gpu # Check if OpenTelemetry is available @@ -72,10 +73,11 @@ def hf_metrics_backend(gh_run): from mellea.backends.cache import SimpleLRUCache from mellea.backends.huggingface import LocalHFBackend - backend = LocalHFBackend( - model_id=IBM_GRANITE_4_1_3B.hf_model_name, # type: ignore - cache=SimpleLRUCache(5), - ) + with hf_skip(): + backend = LocalHFBackend( + model_id=IBM_GRANITE_4_1_3B.hf_model_name, # type: ignore + cache=SimpleLRUCache(5), + ) yield backend From 688df0a9f9f729094e904e438706afc8852e0a19 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Thu, 4 Jun 2026 11:43:40 +0100 Subject: [PATCH 4/4] style(tests): wrap long IntrinsicAdapter call to satisfy ruff-format Assisted-by: Claude Code Signed-off-by: Nigel Jones --- test/backends/test_huggingface.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index bb9805729..c24213732 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -65,7 +65,9 @@ def backend(): model_id=model_ids.IBM_GRANITE_4_1_3B, cache=SimpleLRUCache(5) ) backend.add_adapter( - IntrinsicAdapter("requirement-check", base_model_name=backend.base_model_name) + IntrinsicAdapter( + "requirement-check", base_model_name=backend.base_model_name + ) ) backend.add_adapter( IntrinsicAdapter("answerability", base_model_name=backend.base_model_name)