diff --git a/docs/dev/intrinsics_and_adapters.md b/docs/dev/intrinsics_and_adapters.md index b7be21c2b..3d1375921 100644 --- a/docs/dev/intrinsics_and_adapters.md +++ b/docs/dev/intrinsics_and_adapters.md @@ -13,7 +13,7 @@ These changes are specified by the Adapter that corresponds to a given Intrinsic ## Parts of an Intrinsic Intrinsics specify: -- an adapter name (ie requirement_check) +- an adapter name (ie requirement-check) - types of adapters suitable to be used (ie alora) - any kwargs necessary (ie a requirement like "make sure the last user message is...") diff --git a/docs/dev/requirement_aLoRA_rerouting.md b/docs/dev/requirement_aLoRA_rerouting.md index d4ef25c3b..cdac29cfd 100644 --- a/docs/dev/requirement_aLoRA_rerouting.md +++ b/docs/dev/requirement_aLoRA_rerouting.md @@ -14,7 +14,7 @@ The actual rule is slightly more complicated. ## The Actual Rule -If a `Requirement` is validated using a backend that could either use a `requirement_check` aLoRA or perform an LLMaJ prompt on the underlying model, then the aLoRA is used for validation, even if the `backend.generate_from_context` method is called instead of the `backend._generate_from_intrinsic` method. +If a `Requirement` is validated using a backend that could either use a `requirement-check` aLoRA or perform an LLMaJ prompt on the underlying model, then the aLoRA is used for validation, even if the `backend.generate_from_context` method is called instead of the `backend._generate_from_intrinsic` method. There are three exceptions to this rule: 1. `Backend.default_to_constraint_checking_alora` is set to `False` (this parameter defaults to `True`). @@ -39,8 +39,8 @@ from mellea.backends.adapters import IntrinsicAdapter m = start_session( "huggingface.LocalHFBackend:ibm-granite/granite-4.0-micro") -# By default, the AloraRequirement uses a IntrinsicAdapter with "requirement_check". -m.backend.add_adapter(IntrinsicAdapter("ibm-granite/rag-intrinsics-lib", "requirement_check", base_model_name="granite-4.0-micro")) +# By default, the AloraRequirement uses a IntrinsicAdapter with "requirement-check". +m.backend.add_adapter(IntrinsicAdapter("ibm-granite/rag-intrinsics-lib", "requirement-check", base_model_name="granite-4.0-micro")) m.instruct( "Corporate wants you to find the difference between these two strings:\n\naaa\naba") diff --git a/docs/docs/advanced/intrinsics.md b/docs/docs/advanced/intrinsics.md index a57d1a12a..8fec1b8cd 100644 --- a/docs/docs/advanced/intrinsics.md +++ b/docs/docs/advanced/intrinsics.md @@ -226,7 +226,7 @@ backend = LocalHFBackend(model_id="ibm-granite/granite-4.0-micro") # Register an adapter by task name req_adapter = CustomIntrinsicAdapter( - "requirement_check", + "requirement-check", base_model_name=backend.base_model_name, ) backend.add_adapter(req_adapter) @@ -237,7 +237,7 @@ ctx = ctx.add(Message("assistant", "Yes! What can I help with?")) out, _ = mfuncs.act( Intrinsic( - "requirement_check", + "requirement-check", intrinsic_kwargs={"requirement": "The assistant is helpful."}, ), ctx, @@ -249,4 +249,4 @@ print(out) # {"requirement_likelihood": 1.0} The `Intrinsic` component loads aLoRA adapters (falling back to LoRA) by task name. For OpenAI backends with Granite Switch, adapters are loaded from the model's HuggingFace repository configuration instead of the intrinsic catalog. -Output format is task-specific — `requirement_check` returns a likelihood score. +Output format is task-specific — `requirement-check` returns a likelihood score. diff --git a/docs/examples/intrinsics/README.md b/docs/examples/intrinsics/README.md index b6048c3a4..df3e57e2f 100644 --- a/docs/examples/intrinsics/README.md +++ b/docs/examples/intrinsics/README.md @@ -47,7 +47,7 @@ import mellea.stdlib.functional as mfuncs out, new_ctx = mfuncs.act( Intrinsic( - "requirement_check", + "requirement-check", intrinsic_kwargs={"requirement": "The assistant is helpful."}), ctx, backend diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index fd3d71025..84507299e 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -404,7 +404,7 @@ async def _generate_from_context( if isinstance(action, Requirement): # See docs/dev/requirement_aLoRA_rerouting.md reroute_to_alora = self.default_to_constraint_checking_alora - adapter_name = "requirement_check" + adapter_name = "requirement-check" if isinstance(action, ALoraRequirement): reroute_to_alora = True @@ -416,7 +416,7 @@ async def _generate_from_context( ) alora_action = ALoraRequirement(action.description, adapter_name) - # Check if a requirement_check (or AloraRequirement specified) adapter + # Check if a requirement-check (or AloraRequirement specified) adapter # exists. alora_req_adapter = get_adapter_for_intrinsic( adapter_name, [AdapterType.ALORA], self._added_adapters diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 06fe1c419..1eea93511 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -481,7 +481,7 @@ async def _generate_from_context( # Requirements can be automatically rerouted to a requirement adapter. if isinstance(action, Requirement): reroute_to_alora = self.default_to_constraint_checking_alora - adapter_name = "requirement_check" + adapter_name = "requirement-check" if isinstance(action, ALoraRequirement): reroute_to_alora = True diff --git a/mellea/stdlib/components/intrinsic/rag.py b/mellea/stdlib/components/intrinsic/rag.py index ea77422b8..4dd20a1c8 100644 --- a/mellea/stdlib/components/intrinsic/rag.py +++ b/mellea/stdlib/components/intrinsic/rag.py @@ -198,7 +198,7 @@ def flag_hallucinated_content( documents: collections.abc.Iterable[str | Document], context: ChatContext, backend: AdapterMixin, -) -> float: +) -> list[dict]: """Flag potentially-hallucinated sentences in an agent's response. Intrinsic function that checks whether the sentences in an agent's response to a diff --git a/mellea/stdlib/requirements/requirement.py b/mellea/stdlib/requirements/requirement.py index 098151771..da848b6b8 100644 --- a/mellea/stdlib/requirements/requirement.py +++ b/mellea/stdlib/requirements/requirement.py @@ -35,14 +35,21 @@ def requirement_check_to_bool(x: CBlock | str) -> bool: output = str(x) req_dict: dict[str, Any] = json.loads(output) - likelihood = req_dict.get("requirement_likelihood", None) - if likelihood is None: + likelihood = req_dict.get("requirement_check", None) + if not isinstance(likelihood, dict): MelleaLogger.get_logger().warning( - f"could not get value from alora requirement output; looking for `requirement_likelihood` in {req_dict}" + f"could not get value from alora requirement output; looking for `requirement_check` in {req_dict}" ) return False - if likelihood > 0.5: + score = likelihood.get("score", None) + if score is None: + MelleaLogger.get_logger().warning( + f"could not get value from alora requirement output; looking for `score` in {req_dict}" + ) + return False + + if score > 0.5: return True return False @@ -57,7 +64,7 @@ class ALoraRequirement(Requirement, Intrinsic): Args: description (str): Human-readable requirement description. intrinsic_name (str | None): Name of the ALoRA intrinsic to use. - Defaults to ``"requirement_check"``. + Defaults to ``"requirement-check"``. Attributes: use_aloras (bool): Always ``True``; this class always attempts to use @@ -73,7 +80,7 @@ def __init__(self, description: str, intrinsic_name: str | None = None): self.use_aloras: bool = True if intrinsic_name is None: - intrinsic_name = "requirement_check" + intrinsic_name = "requirement-check" # Initialize the other side of the inheritance tree Intrinsic.__init__( diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index 97bd6b4d4..782112f0a 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -54,17 +54,15 @@ def backend(): """Shared HuggingFace backend for all tests in this module. Uses Granite 3.3-8b for aLoRA adapter compatibility. - The "requirement_check" intrinsic only has adapters for Granite 3.3 models. + The "requirement-check" intrinsic only has adapters for Granite 3.3 models. Granite 4 adapters are not yet available. Other intrinsics are not affected by this issue. """ backend = LocalHFBackend( - model_id="ibm-granite/granite-3.3-8b-instruct", - formatter=TemplateFormatter(model_id="ibm-granite/granite-4.0-tiny-preview"), - cache=SimpleLRUCache(5), + model_id=model_ids.IBM_GRANITE_4_1_3B, cache=SimpleLRUCache(5) ) backend.add_adapter( - IntrinsicAdapter("requirement_check", base_model_name=backend.base_model_name) + IntrinsicAdapter("requirement-check", base_model_name=backend.base_model_name) ) backend.add_adapter( IntrinsicAdapter("answerability", base_model_name=backend.base_model_name) @@ -88,7 +86,7 @@ def session(backend): def test_adapters(backend) -> None: assert len(backend._added_adapters.items()) > 0 - expected_qualified_name = "requirement_check_alora" + expected_qualified_name = "requirement-check_alora" adapter = backend._added_adapters[expected_qualified_name] backend.load_adapter(adapter.qualified_name) assert adapter.qualified_name in backend._loaded_adapters @@ -125,7 +123,7 @@ def test_constraint_lora_with_requirement(session, backend) -> None: assert len(validation_outputs) == 1 val_result = validation_outputs[0] assert isinstance(val_result, ValidationResult) - assert "requirement_likelihood" in str(val_result.reason) + assert "requirement_check" in str(val_result.reason) @pytest.mark.qualitative @@ -158,7 +156,7 @@ def test_constraint_lora_override_does_not_override_alora(session, backend) -> N assert len(validation_outputs) == 1 val_result = validation_outputs[0] assert isinstance(val_result, ValidationResult) - assert "requirement_likelihood" in str(val_result.reason) + assert "requirement_check" in str(val_result.reason) # Ensure the ValidationResult has its thunk and context set. Ensure the context has # the correct actions / results in it. @@ -366,7 +364,7 @@ async def test_generate_with_lock(backend) -> None: b._added_adapters = {} b._loaded_adapters = {} b.add_adapter( - IntrinsicAdapter("requirement_check", base_model_name=b.base_model_name) + IntrinsicAdapter("requirement-check", base_model_name=b.base_model_name) ) b.add_adapter(IntrinsicAdapter("answerability", base_model_name=b.base_model_name)) @@ -397,7 +395,7 @@ def populate_mocked_dict(input_ids, *args, **kwargs): ctx = ChatContext().add(Message("user", "hello")) act = CBlock("hello") raw_act = CBlock("goodb") - req_intrinsic = Intrinsic("requirement_check", {"requirement": "did nothing"}) + req_intrinsic = Intrinsic("requirement-check", {"requirement": "did nothing"}) answerability_intrinsic = Intrinsic("answerability") def call_backend_generate(): @@ -462,7 +460,7 @@ async def test_generate_with_lock_does_not_block_when_awaiting_value(backend) -> # Set up the inputs. ctx = ChatContext().add(Message("user", "hello")) act = CBlock("hello") - req_intrinsic = Intrinsic("requirement_check", {"requirement": "did nothing"}) + req_intrinsic = Intrinsic("requirement-check", {"requirement": "did nothing"}) answerability_intrinsic = Intrinsic("answerability") # Create a few model output thunks: @@ -516,7 +514,7 @@ async def test_generate_with_lock_does_not_block_when_awaiting_value(backend) -> @pytest.mark.qualitative async def test_streaming_error_with_intrinsics(backend) -> None: ctx = ChatContext().add(Message("user", "hello")) - req_intrinsic = Intrinsic("requirement_check", {"requirement": "did nothing"}) + req_intrinsic = Intrinsic("requirement-check", {"requirement": "did nothing"}) with pytest.raises(Exception, match="Intrinsics do not support streaming"): _, _ = await backend.generate_from_context( @@ -540,7 +538,7 @@ async def test_error_during_generate_with_lock(backend) -> None: b._added_adapters = {} b._loaded_adapters = {} b.add_adapter( - IntrinsicAdapter("requirement_check", base_model_name=b.base_model_name) + IntrinsicAdapter("requirement-check", base_model_name=b.base_model_name) ) regular_generate = b._model.generate @@ -559,7 +557,7 @@ def generate_and_raise_exc(*args, **kwargs): # Set up the inputs. ctx = ChatContext().add(Message("user", "hello")) act = CBlock("hello") - req_intrinsic = Intrinsic("requirement_check", {"requirement": "did nothing"}) + req_intrinsic = Intrinsic("requirement-check", {"requirement": "did nothing"}) reg_mot, _ = await b.generate_from_context(act, ctx) req_mot, _ = await b.generate_from_context(req_intrinsic, ctx) @@ -753,7 +751,7 @@ def get_temperature(location: str) -> int: decoded = backend._tokenizer.decode(input_tokens[0]) print(decoded) assert "get_temperature" in decoded - assert "available_tools" in decoded, ( + assert "access to the following tools" in decoded, ( "expected string from system prompt with tool calls not found; if you changed the model, that might have caused this issue" ) diff --git a/test/stdlib/requirements/test_requirement.py b/test/stdlib/requirements/test_requirement.py index 125022e06..0e9e0956a 100644 --- a/test/stdlib/requirements/test_requirement.py +++ b/test/stdlib/requirements/test_requirement.py @@ -76,16 +76,16 @@ def test_simple_validate_invalid(): def test_requirement_check_to_bool_above_threshold(): - assert requirement_check_to_bool('{"requirement_likelihood": 0.8}') is True + assert requirement_check_to_bool('{"requirement_check": {"score": 0.8}}') is True def test_requirement_check_to_bool_below_threshold(): - assert requirement_check_to_bool('{"requirement_likelihood": 0.3}') is False + assert requirement_check_to_bool('{"requirement_check": {"score":0.3}}') is False def test_requirement_check_to_bool_at_threshold(): """0.5 is NOT > 0.5, so should return False.""" - assert requirement_check_to_bool('{"requirement_likelihood": 0.5}') is False + assert requirement_check_to_bool('{"requirement_check": {"score": 0.5}}') is False def test_requirement_check_to_bool_missing_key(): @@ -163,7 +163,7 @@ def test_alora_requirement_default_intrinsic(mock_intrinsic_init): # Intrinsic.__init__ is unbound; mock receives self as first positional arg. mock_intrinsic_init.assert_called_once_with( r, - intrinsic_name="requirement_check", + intrinsic_name="requirement-check", intrinsic_kwargs={"requirement": "must be valid"}, ) diff --git a/uv.lock b/uv.lock index 5908de1cb..4e2a92295 100644 --- a/uv.lock +++ b/uv.lock @@ -3612,7 +3612,7 @@ requires-dist = [ { name = "trl", marker = "extra == 'hf'", specifier = "==0.19.1" }, { name = "typer", marker = "extra == 'cli'" }, { name = "uvicorn", marker = "extra == 'server'" }, - { name = "xgrammar", marker = "extra == 'hf'" }, + { name = "xgrammar", marker = "extra == 'hf'", specifier = "==0.1.33" }, ] provides-extras = ["hf", "litellm", "watsonx", "tools", "telemetry", "docling", "granite-retriever", "cli", "server", "sandbox", "switch", "backends", "hooks", "all"]