diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index e6a8d5236..c24213732 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -47,6 +47,7 @@ from mellea.stdlib.components import Intrinsic, Message from mellea.stdlib.context import ChatContext, SimpleContext from mellea.stdlib.requirements import ALoraRequirement, LLMaJRequirement +from test.conftest import hf_skip @pytest.fixture(scope="module") @@ -59,15 +60,18 @@ def backend(): against this granite-3.3 backend will fail once revision pinning is wired through in phase-2.2 (#1141). Other intrinsics are not affected. """ - backend = LocalHFBackend( - model_id=model_ids.IBM_GRANITE_4_1_3B, cache=SimpleLRUCache(5) - ) - backend.add_adapter( - IntrinsicAdapter("requirement-check", base_model_name=backend.base_model_name) - ) - backend.add_adapter( - IntrinsicAdapter("answerability", base_model_name=backend.base_model_name) - ) + with hf_skip(): + backend = LocalHFBackend( + model_id=model_ids.IBM_GRANITE_4_1_3B, cache=SimpleLRUCache(5) + ) + backend.add_adapter( + IntrinsicAdapter( + "requirement-check", base_model_name=backend.base_model_name + ) + ) + backend.add_adapter( + IntrinsicAdapter("answerability", base_model_name=backend.base_model_name) + ) yield backend from test.conftest import cleanup_gpu_backend diff --git a/test/backends/test_huggingface_tools.py b/test/backends/test_huggingface_tools.py index b2a72176f..aa658ac1f 100644 --- a/test/backends/test_huggingface_tools.py +++ b/test/backends/test_huggingface_tools.py @@ -25,14 +25,16 @@ from mellea.backends.cache import SimpleLRUCache from mellea.backends.huggingface import LocalHFBackend from mellea.stdlib.context import ChatContext +from test.conftest import hf_skip @pytest.fixture(scope="module") def backend(): """Shared HuggingFace backend for all tests in this module.""" - backend = LocalHFBackend( - model_id=model_ids.MISTRALAI_MISTRAL_0_3_7B, cache=SimpleLRUCache(5) - ) + with hf_skip(): + backend = LocalHFBackend( + model_id=model_ids.MISTRALAI_MISTRAL_0_3_7B, cache=SimpleLRUCache(5) + ) # add_granite_aloras(backend) yield backend diff --git a/test/conftest.py b/test/conftest.py index cf6fed7ae..d424d5e15 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,3 +1,4 @@ +import contextlib import gc import os import subprocess @@ -9,6 +10,30 @@ from mellea.core import MelleaLogger +# ============================================================================ +# HuggingFace Hub Skip Helper +# ============================================================================ + + +@contextlib.contextmanager +def hf_skip(): + """Skip the current test/fixture when a HuggingFace Hub call fails. + + Catches OSError (covers HfHubHTTPError, LocalEntryNotFoundError, and + transformers-wrapped Hub errors) plus requests network errors. Use this + in every fixture that downloads model weights or YAML files from the Hub + so a 429 or transient network blip results in a clean skip rather than an + ERROR that masks unrelated failures. + """ + try: + yield + except (OSError, requests.exceptions.RequestException) as e: + pytest.skip( + f"HuggingFace Hub not accessible: {type(e).__name__}: {e}", + allow_module_level=True, + ) + + # Try to import optional dependencies for system detection try: import psutil diff --git a/test/formatters/granite/test_intrinsics_formatters.py b/test/formatters/granite/test_intrinsics_formatters.py index 2231255a6..e52d49509 100644 --- a/test/formatters/granite/test_intrinsics_formatters.py +++ b/test/formatters/granite/test_intrinsics_formatters.py @@ -21,7 +21,6 @@ torch = pytest.importorskip("torch", reason="torch not installed — install mellea[hf]") import yaml -from huggingface_hub.errors import LocalEntryNotFoundError # First Party from mellea.formatters.granite import ( @@ -32,6 +31,7 @@ ) from mellea.formatters.granite.base import util as base_util from mellea.formatters.granite.intrinsics import json_util, util as intrinsics_util +from test.conftest import hf_skip def _read_file(name): @@ -147,15 +147,13 @@ def _resolve_yaml(self): object. Called at fixture creation (execution time) to prevent collection time errors. """ if not self.yaml_file: - try: + with hf_skip(): self.yaml_file = intrinsics_util.obtain_io_yaml( self.task, self.base_model_id, self.repo_id, revision=self.revision, # type: ignore ) - except (LocalEntryNotFoundError, requests.exceptions.RequestException) as e: - pytest.skip(f"HuggingFace Hub not accessible: {type(e).__name__}: {e}") return self @@ -537,12 +535,10 @@ def test_read_yaml(): IntrinsicsRewriter(config_file=_INPUT_YAML_DIR / "answerability.yaml") # Read from Hugging Face hub. - try: + with hf_skip(): local_path = intrinsics_util.obtain_io_yaml( "answerability", "granite-4.0-micro", _RAG_INTRINSICS_REPO_NAME ) - except (LocalEntryNotFoundError, requests.exceptions.RequestException) as e: - pytest.skip(f"HuggingFace Hub not accessible: {type(e).__name__}: {e}") IntrinsicsRewriter(config_file=local_path) diff --git a/test/stdlib/components/docs/test_richdocument.py b/test/stdlib/components/docs/test_richdocument.py index b6011d7cd..06e861a4d 100644 --- a/test/stdlib/components/docs/test_richdocument.py +++ b/test/stdlib/components/docs/test_richdocument.py @@ -11,6 +11,7 @@ import mellea from mellea.core import TemplateRepresentation from mellea.stdlib.components.docs.richdocument import RichDocument, Table +from test.conftest import hf_skip from test.predicates import require_gpu pytestmark = pytest.mark.integration @@ -20,9 +21,10 @@ def rd() -> RichDocument: # Use a specific document so we can test some of the functionality # related to extracting and transforming text. - return RichDocument.from_document_file( - "https://arxiv.org/pdf/1906.04043", do_ocr=False - ) + with hf_skip(): + return RichDocument.from_document_file( + "https://arxiv.org/pdf/1906.04043", do_ocr=False + ) def test_richdocument_basics(rd: RichDocument): diff --git a/test/stdlib/components/intrinsic/test_core.py b/test/stdlib/components/intrinsic/test_core.py index 3c8df0eb9..b6724512d 100644 --- a/test/stdlib/components/intrinsic/test_core.py +++ b/test/stdlib/components/intrinsic/test_core.py @@ -17,7 +17,7 @@ from mellea.stdlib.components.intrinsic import core from mellea.stdlib.components.intrinsic._util import call_intrinsic from mellea.stdlib.context import ChatContext -from test.conftest import cleanup_gpu_backend +from test.conftest import cleanup_gpu_backend, hf_skip from test.predicates import require_gpu from test.stdlib.components.intrinsic.test_rag import ( _read_input_json as _read_rag_input_json, @@ -48,7 +48,8 @@ def _backend(): # Prevent thrashing if the default device is CPU torch.set_num_threads(4) - backend_ = LocalHFBackend(model_id=BASE_MODEL) + with hf_skip(): + backend_ = LocalHFBackend(model_id=BASE_MODEL) yield backend_ # Code after yield is cleanup code. diff --git a/test/stdlib/components/intrinsic/test_guardian.py b/test/stdlib/components/intrinsic/test_guardian.py index 04d3e9214..3e3663be4 100644 --- a/test/stdlib/components/intrinsic/test_guardian.py +++ b/test/stdlib/components/intrinsic/test_guardian.py @@ -14,7 +14,7 @@ from mellea.stdlib.components import Document, Message from mellea.stdlib.components.intrinsic import guardian from mellea.stdlib.context import ChatContext -from test.conftest import cleanup_gpu_backend +from test.conftest import cleanup_gpu_backend, hf_skip from test.predicates import require_gpu # Skip entire module in CI since all tests are qualitative @@ -37,7 +37,8 @@ def _backend(): """Backend used by the tests in this file. Module-scoped to avoid reloading the model for each test.""" torch.set_num_threads(4) - backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_MICRO_3B.hf_model_name) # type: ignore + with hf_skip(): + backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_MICRO_3B.hf_model_name) # type: ignore yield backend_ cleanup_gpu_backend(backend_, "test_guardian") diff --git a/test/stdlib/components/intrinsic/test_rag.py b/test/stdlib/components/intrinsic/test_rag.py index 3bfc591bf..f0a85f2ce 100644 --- a/test/stdlib/components/intrinsic/test_rag.py +++ b/test/stdlib/components/intrinsic/test_rag.py @@ -15,6 +15,7 @@ from mellea.stdlib.components import Document, Message from mellea.stdlib.components.intrinsic import rag from mellea.stdlib.context import ChatContext +from test.conftest import hf_skip from test.predicates import require_gpu # Skip entire module in CI since all 7 tests are qualitative @@ -42,7 +43,8 @@ def _backend(): torch.set_num_threads(4) # No adapters for hybrid version. - backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_1_3B.hf_model_name) + with hf_skip(): + backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_1_3B.hf_model_name) yield backend_ from test.conftest import cleanup_gpu_backend @@ -57,7 +59,8 @@ def _backend_4_0(): torch.set_num_threads(4) # No adapters for hybrid version. - backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_MICRO_3B.hf_model_name) + with hf_skip(): + backend_ = LocalHFBackend(model_id=IBM_GRANITE_4_MICRO_3B.hf_model_name) yield backend_ from test.conftest import cleanup_gpu_backend diff --git a/test/stdlib/requirements/test_groundedness_requirement.py b/test/stdlib/requirements/test_groundedness_requirement.py index 2d487b96a..d9f912e5b 100644 --- a/test/stdlib/requirements/test_groundedness_requirement.py +++ b/test/stdlib/requirements/test_groundedness_requirement.py @@ -9,13 +9,15 @@ from mellea.stdlib.components import Document, Message from mellea.stdlib.context import ChatContext from mellea.stdlib.requirements.rag import GroundednessRequirement +from test.conftest import hf_skip from test.predicates import require_gpu @pytest.fixture def backend(): """Provide HuggingFace backend for tests.""" - return LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") + with hf_skip(): + return LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") @pytest.fixture diff --git a/test/stdlib/requirements/test_groundedness_requirement_e2e.py b/test/stdlib/requirements/test_groundedness_requirement_e2e.py index fbbe33e6a..2d4b9a7e1 100644 --- a/test/stdlib/requirements/test_groundedness_requirement_e2e.py +++ b/test/stdlib/requirements/test_groundedness_requirement_e2e.py @@ -6,13 +6,15 @@ from mellea.stdlib.components import Document, Message from mellea.stdlib.context import ChatContext from mellea.stdlib.requirements.rag import GroundednessRequirement +from test.conftest import hf_skip from test.predicates import require_gpu @pytest.fixture(scope="session") def backend(): """Provide HuggingFace backend for tests (session-scoped to avoid OOM).""" - return LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") + with hf_skip(): + return LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") @pytest.fixture diff --git a/test/stdlib/test_spans.py b/test/stdlib/test_spans.py index 7914fa02a..1946025e2 100644 --- a/test/stdlib/test_spans.py +++ b/test/stdlib/test_spans.py @@ -10,6 +10,7 @@ from mellea.core import CBlock from mellea.stdlib.components import SimpleComponent from mellea.stdlib.session import MelleaSession, start_session +from test.conftest import hf_skip from test.predicates import require_gpu # Module-level markers for all tests using Granite 4.1 3B model @@ -19,11 +20,12 @@ # We edit the context type in the async tests below. Don't change the scope here. @pytest.fixture(scope="function") def m_session(gh_run): - m = start_session( - "hf", - model_id=IBM_GRANITE_4_1_3B, - model_options={ModelOption.MAX_NEW_TOKENS: 64}, - ) + with hf_skip(): + m = start_session( + "hf", + model_id=IBM_GRANITE_4_1_3B, + model_options={ModelOption.MAX_NEW_TOKENS: 64}, + ) yield m from test.conftest import cleanup_gpu_backend diff --git a/test/telemetry/test_metrics_backend.py b/test/telemetry/test_metrics_backend.py index c538accd2..2119501cf 100644 --- a/test/telemetry/test_metrics_backend.py +++ b/test/telemetry/test_metrics_backend.py @@ -16,6 +16,7 @@ ) from mellea.stdlib.components import Message from mellea.stdlib.context import SimpleContext +from test.conftest import hf_skip from test.predicates import require_api_key, require_gpu # Check if OpenTelemetry is available @@ -72,10 +73,11 @@ def hf_metrics_backend(gh_run): from mellea.backends.cache import SimpleLRUCache from mellea.backends.huggingface import LocalHFBackend - backend = LocalHFBackend( - model_id=IBM_GRANITE_4_1_3B.hf_model_name, # type: ignore - cache=SimpleLRUCache(5), - ) + with hf_skip(): + backend = LocalHFBackend( + model_id=IBM_GRANITE_4_1_3B.hf_model_name, # type: ignore + cache=SimpleLRUCache(5), + ) yield backend