From 690e1eeb9bb8a79a4cf7a7d8a99dadc8ba8fb995 Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Sat, 25 Oct 2025 16:58:32 -0400 Subject: [PATCH 01/38] test: remove hf .to(device) --- .../aloras/huggingface/granite_aloras.py | 13 +++---- mellea/backends/huggingface.py | 36 ++++--------------- .../process_reward_models/huggingface/prms.py | 6 ++-- 3 files changed, 14 insertions(+), 41 deletions(-) diff --git a/mellea/backends/aloras/huggingface/granite_aloras.py b/mellea/backends/aloras/huggingface/granite_aloras.py index b5e29a475..baffaa847 100644 --- a/mellea/backends/aloras/huggingface/granite_aloras.py +++ b/mellea/backends/aloras/huggingface/granite_aloras.py @@ -96,7 +96,7 @@ def generate_using_strings( # Get the constraint tokens separately so that we can calculate the alora offsets. constraint_tokens = self._backend._tokenizer( self._constraint_prompt.format(constraint), return_tensors="pt" - ).to(self._backend._device) + ) alora_offsets = [ constraint_tokens["input_ids"].shape[1] @@ -106,8 +106,8 @@ def generate_using_strings( chat_response = asyncio.to_thread( self._backend.alora_model.generate, - input_combined["input_ids"].to(self._backend._device), - attention_mask=input_combined["attention_mask"].to(self._backend._device), + input_combined["input_ids"], + attention_mask=input_combined["attention_mask"], max_new_tokens=1, return_dict_in_generate=True, alora_offsets=alora_offsets, @@ -149,7 +149,7 @@ def _generate_using_cache( # Must tokenize the constraint here since the requirement isn't known at initialization. constraint_tokens = self._backend._tokenizer( self._constraint_prompt.format(constraint), return_tensors="pt" - ).to(self._backend._device) + ) input_combined = { "input_ids": torch.cat( @@ -196,10 +196,7 @@ def _generate_not_using_cache( # Must tokenize the constraint here since the requirement isn't known at initialization. templatized = templatized + self._constraint_prompt.format(constraint) - tokenized = self._backend._tokenizer(templatized, return_tensors="pt").to( - self._backend._device - ) - + tokenized = self._backend._tokenizer(templatized, return_tensors="pt") input_combined = { "input_ids": torch.cat( [tokenized["input_ids"], self._generation_prompt_tokens["input_ids"]], diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index fb9cacf6f..e926b0907 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -148,23 +148,15 @@ def __init__( self._hf_model_id = model_id.hf_model_name match custom_config: case None: - # Choose a device. - self._device = torch.device( - "cuda" - if torch.cuda.is_available() - else "mps" - if torch.backends.mps.is_available() - else "cpu" - ) # Get the model and tokenizer. self._model: PreTrainedModel = AutoModelForCausalLM.from_pretrained( self._hf_model_id - ).to(self._device) # type: ignore + ) self._tokenizer: PreTrainedTokenizer = AutoTokenizer.from_pretrained( self._hf_model_id ) case _: - self._tokenizer, self._model, self._device = custom_config + self._tokenizer, self._model, _ = custom_config self._use_caches = use_caches self._cache = cache if cache is not None else SimpleLRUCache(3) @@ -337,7 +329,7 @@ def _generate_from_context_standard( add_generation_prompt=True, # If we change this, must modify huggingface granite guardian. return_tensors="pt", **self._make_backend_specific_and_remove(model_options), - ).to(self._device) # type: ignore + ) format_kwargs = {} if _format: @@ -489,7 +481,7 @@ async def post_processing( cache_info = HFAloraCacheInfo( kv_cache=cache, merged_token_ids=output_complete, - merged_attention=torch.ones_like(output_complete).to(self._device), + merged_attention=torch.ones_like(output_complete), q_end=len(input_ids[0]), # type: ignore ) @@ -543,9 +535,7 @@ def _generate_from_raw( prompts = [self.formatter.print(action) for action in actions] # batch-encoding call is deprecated in favor of this - inputs = self._tokenizer(prompts, return_tensors="pt", padding=True).to( - self._device - ) + inputs = self._tokenizer(prompts, return_tensors="pt", padding=True) if format is None: outputs = self._model.generate( # type: ignore @@ -787,7 +777,7 @@ def __init__( self._generation_prompt = generation_prompt self._generation_prompt_tokens = self._backend._tokenizer( self._generation_prompt, return_tensors="pt" - ).to(self._backend._device) + ) class HFProcessRewardModel(PRM, abc.ABC): @@ -805,23 +795,9 @@ def __init__( """ super().__init__(model_name_or_path) - # auto-device if not more specific - self._device = device - if device is None: - device_name: str = ( - "cuda" - if torch.cuda.is_available() - else "mps" - if torch.backends.mps.is_available() - else "cpu" - ) - assert device_name is not None - self._device = torch.device(device_name) # type: ignore - self.model: PreTrainedModel = AutoModelForCausalLM.from_pretrained( self.model_name_or_path, torch_dtype=torch.bfloat16 ) - self.model.to(self._device) # type: ignore self.model.eval() self.tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_path) diff --git a/mellea/backends/process_reward_models/huggingface/prms.py b/mellea/backends/process_reward_models/huggingface/prms.py index 2525b8e6a..c8c7be780 100644 --- a/mellea/backends/process_reward_models/huggingface/prms.py +++ b/mellea/backends/process_reward_models/huggingface/prms.py @@ -71,7 +71,7 @@ def score(self, query: str, response: str) -> tuple[list[float], list[list[float # move each item of the batch to the device for i in batches: - batches[i] = batches[i].to(self.model.device) + batches[i] = batches[i] with torch.no_grad(): model_outputs = self.model(**batches) @@ -178,7 +178,7 @@ def __init__( # initialize PRM head self.prm_head = torch.nn.Linear( self.model.config.hidden_size, 2, bias=False, dtype=self.model.dtype - ).to(self.model.device) + ) state = torch.load(model_name_or_path + "/added_params.bin") # need to do this-- we save model dict as `prm_head.weight` during training @@ -205,7 +205,7 @@ def score(self, query: str, response: str) -> tuple[list[float], list[list[float batch = self.prepare_inputs(query, list_of_steps) # move each item of the batch to the device for i in batch: - batch[i] = batch[i].to(self.model.device) + batch[i] = batch[i] with torch.no_grad(): model_outputs = self.model(**batch, output_hidden_states=True) From cdca73a31567d60ee587ff188b7cf88325212fed Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Sat, 25 Oct 2025 17:06:31 -0400 Subject: [PATCH 02/38] fix: remove alora model field from hf --- .../aloras/huggingface/granite_aloras.py | 5 ++-- mellea/backends/huggingface.py | 29 ++----------------- 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/mellea/backends/aloras/huggingface/granite_aloras.py b/mellea/backends/aloras/huggingface/granite_aloras.py index baffaa847..2be4c3901 100644 --- a/mellea/backends/aloras/huggingface/granite_aloras.py +++ b/mellea/backends/aloras/huggingface/granite_aloras.py @@ -61,12 +61,11 @@ def generate_using_strings( stream: bool = False, ) -> ModelOutputThunk: """Generates a constraint response from the ALora. Must be run in a running event loop.""" - assert self._backend.alora_model is not None # Go ahead and do runtime type-checking because passing CBlocks into this function is a common error. assert type(input) is str assert type(response) is str assert type(constraint) is str - self._backend.alora_model.set_adapter(self.name) + self._backend._model.set_adapter(self.name) cache_hit = self._backend.cache_get(response) if stream: @@ -105,7 +104,7 @@ def generate_using_strings( ] chat_response = asyncio.to_thread( - self._backend.alora_model.generate, + self._backend._model.generate, # type: ignore input_combined["input_ids"], attention_mask=input_combined["attention_mask"], max_new_tokens=1, diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index e926b0907..29dd3144d 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -56,9 +56,6 @@ from mellea.stdlib.chat import Message from mellea.stdlib.requirement import ALoraRequirement, LLMaJRequirement, Requirement -if TYPE_CHECKING: - from alora.peft_model_alora import aLoRAPeftModelForCausalLM # type: ignore - assert outlines, "outlines needs to be present to make outlines_core work" """A configuration type for the unhappy path: Tokenizer * Model * torch device string @@ -161,22 +158,8 @@ def __init__( self._use_caches = use_caches self._cache = cache if cache is not None else SimpleLRUCache(3) - # Used when running aLoRAs with this backend. - self._alora_model: "aLoRAPeftModelForCausalLM | None" = None # noqa: UP037 - # ALoras that have been loaded for this model. self._aloras: dict[str, HFAlora] = {} - @property - def alora_model(self) -> "aLoRAPeftModelForCausalLM | None": # noqa: UP037 - """The ALora model.""" - return self._alora_model - - @alora_model.setter - def alora_model(self, model: "aLoRAPeftModelForCausalLM | None"): # noqa: UP037 - """Sets the ALora model. This should only happen once in a backend's lifetime.""" - assert self._alora_model is None - self._alora_model = model - def generate_from_context( self, action: Component | CBlock, @@ -719,8 +702,6 @@ def add_alora(self, alora: HFAlora): Args: alora (str): identifier for the ALora adapter """ - from alora.peft_model_alora import aLoRAPeftModelForCausalLM # type: ignore - assert issubclass(alora.__class__, HFAlora), ( f"cannot add an ALora of type {alora.__class__} to model; must inherit from {HFAlora.__class__}" ) @@ -732,14 +713,8 @@ def add_alora(self, alora: HFAlora): ) return None - if self.alora_model is None: - base_model = self._model - self.alora_model = aLoRAPeftModelForCausalLM.from_pretrained( - base_model, alora.path_or_model_id, alora.name - ) - else: - self.alora_model.load_adapter(alora.path_or_model_id, alora.name) - + self._model.load_adapter(alora.path_or_model_id, alora.name) + self._model.disable_adapters() self._aloras[alora.name] = alora def get_alora(self, alora_name: str) -> Alora | None: From 3a1c1e8443cab13fbc9239030d90036911da3549 Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Sun, 26 Oct 2025 17:44:08 -0400 Subject: [PATCH 03/38] test: init commit; need to refactor a bunch --- mellea/backends/adapters/adapter.py | 134 +++++++++ .../aloras/huggingface/granite_aloras.py | 28 ++ .../backends/aloras/openai/granite_aloras.py | 33 ++- mellea/backends/huggingface.py | 166 ++++++++++- mellea/backends/openai.py | 258 ++++++++++++++++-- mellea/backends/types.py | 23 ++ mellea/stdlib/intrinsics/intrinsic.py | 100 +++++++ pyproject.toml | 6 +- uv.lock | 27 +- 9 files changed, 731 insertions(+), 44 deletions(-) create mode 100644 mellea/backends/adapters/adapter.py create mode 100644 mellea/stdlib/intrinsics/intrinsic.py diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py new file mode 100644 index 000000000..88914df19 --- /dev/null +++ b/mellea/backends/adapters/adapter.py @@ -0,0 +1,134 @@ +import abc +from enum import Enum + +import granite_common + +from mellea.backends.types import _ServerType + + +class AdapterType(Enum): + LORA = "lora" + ALORA = "alora" + + +class Adapter(abc.ABC): + def __init__(self, name: str, adapter_type: AdapterType): + """An adapter that can be added to a backend. + + Args: + name: name of the adapter; when referencing this adapter, use adapter.qualified_name + adapter_type: enum describing what type of adapter it is (ie LORA / ALORA) + """ + self.name = name + self.adapter_type = adapter_type + self.qualified_name = name + "_" + adapter_type.value + """the name of the adapter to use when loading / looking it up""" + + # @abc.abstractmethod + # def add_adapter(self, backend: Backend, *args, **kwargs): + # """Adds an adapter.""" + + # def already_added(self, backend: Backend) -> bool: + # """Checks if a this adapter has already been loaded to a backend and if so, if it's the current backend. + + # Args: + # backend: the Backend to add the adapter to. + + # Returns: + # True if already added; False if not already added. + + # Raises: + # ValueError: if the adapter has already been added to a different backend. + # """ + # if self.backend is not None: + # if self.backend is not backend: + # raise ValueError(f"adapter ({self.name}, {self}) has already been added to a different backend ({self.backend})") + # else: + # # It's already been added to this backend. Do nothing. + # return True + # return False + + +class OpenAIAdapter(Adapter): + @abc.abstractmethod + def get_open_ai_path( + self, + base_model_name: str, + server_type: _ServerType = _ServerType.LOCALHOST, + remote_path: str | None = None, + ) -> str: + """Returns the path needed to load the adapter. + + Args: + base_model_name: the base model; typically the last part of the huggingface model id like "granite-3.3-8b-instruct" + server_type: the server type (ie LOCALHOST / OPENAI); usually the backend has information on this + remote_path: optional; used only if the server_type is REMOTE_VLLM; base path at which to find the adapter + """ + ... + + +class LocalHFAdapter(Adapter): + @abc.abstractmethod + def get_local_hf_path(self, base_model_name: str) -> str: + """Returns the path needed to load the adapter. + + Args: + base_model_name: the base model; typically the last part of the huggingface model id like "granite-3.3-8b-instruct" + """ + ... + + +class RagIntrinsicAdapter(OpenAIAdapter, LocalHFAdapter): + def __init__(self, name: str, adapter_type: AdapterType = AdapterType.ALORA): + """An adapter that can be added to either an `OpenAIBackend` or a `LocalHFBackend`. Most rag-lib-intrinsics support lora or alora adapter types. + + Args: + name: name of the adapter; when referencing this adapter, use adapter.qualified_name + adapter_type: enum describing what type of adapter it is (ie LORA / ALORA) + """ + assert adapter_type == AdapterType.ALORA or adapter_type == AdapterType.LORA, ( + f"{adapter_type} not supported" + ) + super().__init__(name, adapter_type) + + def get_open_ai_path( + self, + base_model_name: str, + server_type: _ServerType = _ServerType.LOCALHOST, + remote_path: str | None = None, + ) -> str: + if server_type == _ServerType.LOCALHOST: + path = self.download_and_get_path(base_model_name) + elif server_type == _ServerType.REMOTE_VLLM: + if remote_path is None: + remote_path = "rag-intrinsics-lib" + path = self.get_path_on_remote(base_model_name, remote_path) + else: + raise ValueError( + f"{self} not supported for OpenAIBackend with server_type: {server_type}" + ) + + return path + + def get_local_hf_path(self, base_model_name: str) -> str: + return self.download_and_get_path(base_model_name) + + def download_and_get_path(self, base_model_name: str) -> str: + """Downloads the required rag intrinsics files if necessary and returns the path to the them. + + Args: + base_model_name: the base model; typically the last part of the huggingface model id like "granite-3.3-8b-instruct" + + Returns: + a path to the files + """ + is_alora = self.adapter_type == AdapterType.ALORA + return str( + granite_common.intrinsics.util.obtain_lora( + self.name, base_model_name, alora=is_alora + ) + ) + + def get_path_on_remote(self, base_model_name: str, base_path: str) -> str: + """Assumes the files have already been downloaded on the remote server.""" + return f"./{base_path}/{self.name}/{self.adapter_type.value}/{base_model_name}" diff --git a/mellea/backends/aloras/huggingface/granite_aloras.py b/mellea/backends/aloras/huggingface/granite_aloras.py index 2be4c3901..22809ed97 100644 --- a/mellea/backends/aloras/huggingface/granite_aloras.py +++ b/mellea/backends/aloras/huggingface/granite_aloras.py @@ -4,6 +4,7 @@ import functools from copy import deepcopy +import granite_common import torch from transformers.generation.utils import GenerateDecoderOnlyOutput @@ -279,3 +280,30 @@ def add_granite_aloras(backend: LocalHFBackend): raise ValueError( f"cannot add_granite_aloras to unknown huggingface model_id / backend: {backend._hf_model_id}" ) + + +class HFIntrinsicAlora(HFAlora): + def __init__( + self, name: str, backend: LocalHFBackend, base_model_name: str | None = None + ): + # TODO: JAL. May need to change HFAlora to not need a generation prompt by default... + + if base_model_name is None: + # TODO: JAL. Make sure all hf model ids fit this way. + base_model_name = backend._hf_model_id.split("/")[1] + + # TODO: JAL. figure out if we want to download files here or when adding the alora... + # Need to do all the path determination at init? + path = str( + granite_common.intrinsics.util.obtain_lora( + name, base_model_name, alora=True + ) + ) + + super().__init__(name, path, "", backend) + + # We do a lot of logging for ALoras because this is an experimental feature. Maybe we should tag these log messages? + self._logger = FancyLogger.get_logger() + + def generate_using_strings(self, *args, **kwargs) -> ModelOutputThunk: + raise NotImplementedError() diff --git a/mellea/backends/aloras/openai/granite_aloras.py b/mellea/backends/aloras/openai/granite_aloras.py index 6d1b2c6bc..a53e0f1ca 100644 --- a/mellea/backends/aloras/openai/granite_aloras.py +++ b/mellea/backends/aloras/openai/granite_aloras.py @@ -16,17 +16,18 @@ from mellea.stdlib.base import GenerateType, ModelOutputThunk -class OpenAIConstraintAlora(OpenAIAlora): +# TODO: JAL. this should be made into a generic adapter interface...? +class OpenAIIntrinsicAlora(OpenAIAlora): """The [Requirement Checking ALora for Granite 3.2 8B](https://huggingface.co/ibm-granite/granite-3.2-8b-alora-requirement-check) checks if the specified requirement was satisfied by the most recent model generation. Only one requirement is checked at a time.""" def __init__( self, name: str, path: str, generation_prompt: str, backend: OpenAIBackend ): """Initialize after checking that the backend is correct.""" - assert backend._hf_model_id == "ibm-granite/granite-3.2-8b-instruct" super().__init__(name, path, generation_prompt, backend) # We do a lot of logging for ALoras because this is an experimental feature. Maybe we should tag these log messages? self._logger = FancyLogger.get_logger() + # TODO: JAL. Move the adapter loading logic to the init?... or see where we want to make sure they are downloaded... def generate_using_strings( self, @@ -118,11 +119,25 @@ async def post_processing(backend: OpenAIBackend, mot: ModelOutputThunk): def add_granite_aloras(backend: OpenAIBackend): """Adds the IBM Granite "starter pack" ALoras to a backend.""" - backend.add_alora( - OpenAIConstraintAlora( - name="constraint", - path="ibm-granite/granite-3.2-8b-alora-requirement-check", - generation_prompt="<|start_of_role|>check_requirement<|end_of_role|>", - backend=backend, + if backend._hf_model_id == "ibm-granite/granite-3.2-8b-instruct": + backend.add_alora( + OpenAIIntrinsicAlora( + name="constraint", + path="ibm-granite/granite-3.2-8b-alora-requirement-check", + generation_prompt="<|start_of_role|>check_requirement<|end_of_role|>", + backend=backend, + ) + ) + elif backend._hf_model_id == "ibm-granite/granite-3.3-8b-instruct": + backend.add_alora( + OpenAIIntrinsicAlora( + name="constraint", + path="ibm-granite/granite-3.3-8b-alora-requirement-check", + generation_prompt="<|start_of_role|>check_requirement<|end_of_role|>", + backend=backend, + ) + ) + else: + raise ValueError( + f"cannot add_granite_aloras to unknown huggingface model_id / backend: {backend._hf_model_id}" ) - ) diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index 29dd3144d..c19ff1524 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -13,10 +13,13 @@ import inspect import json from collections.abc import Callable, Coroutine -from typing import TYPE_CHECKING, Any +from copy import deepcopy +from typing import TYPE_CHECKING, Any, cast +import granite_common import outlines import outlines_core +import peft import torch from transformers import ( AsyncTextIteratorStreamer, @@ -30,10 +33,12 @@ from transformers.generation.utils import GenerateDecoderOnlyOutput from mellea.backends import BaseModelSubclass +from mellea.backends.adapters.adapter import LocalHFAdapter from mellea.backends.aloras import Alora, AloraBackendMixin from mellea.backends.cache import Cache, SimpleLRUCache from mellea.backends.formatter import Formatter, FormatterBackend, TemplateFormatter from mellea.backends.model_ids import ModelIdentifier +from mellea.backends.openai import OpenAIBackend from mellea.backends.process_reward_models import PRM from mellea.backends.tools import ( add_tools_from_context_actions, @@ -54,6 +59,7 @@ ModelToolCall, ) from mellea.stdlib.chat import Message +from mellea.stdlib.intrinsics.intrinsic import Intrinsic from mellea.stdlib.requirement import ALoraRequirement, LLMaJRequirement, Requirement assert outlines, "outlines needs to be present to make outlines_core work" @@ -189,6 +195,12 @@ def generate_from_context( action, ctx, _format=format, model_options=model_opts ) return mot, ctx.add(mot) + + elif isinstance(action, Intrinsic): + return self._generate_from_intrinsic( + action, ctx, _format=format, model_options=model_opts + ) + mot = self._generate_from_context_standard( action, ctx, _format=format, model_options=model_opts, tool_calls=tool_calls ) @@ -241,6 +253,144 @@ def _generate_from_context_alora( return alora_output + def _generate_from_intrinsic( + self, + action: Intrinsic, + ctx: Context, + *, + _format: type[BaseModelSubclass] | None = None, # TODO: JAL. Remove this param? + model_options: dict[str, Any], + tool_calls: bool = False, # TODO: JAL. Remove this param? + ) -> ModelOutputThunk: + if not ctx.is_chat_context: + raise Exception("Does not yet support non-chat contexts.") + + linearized_ctx = ctx.view_for_generation() + assert linearized_ctx is not None, ( + "If ctx.is_chat_context, then the context should be linearizable." + ) + ctx_as_message_list: list[Message] = self.formatter.to_chat_messages( + linearized_ctx + ) + + # TODO: JAL. Fix where this happens. + # add action + # ctx_as_message_list.extend(self.formatter.to_chat_messages([action])) + # ctx_as_conversation = [ + # {"role": m.role, "content": m.content} for m in ctx_as_message_list + # ] + + # Check that we ddin't accidentally end up with CBlocks. + # for msg in ctx_as_conversation: + # for v in msg.values(): + # if "CBlock" in v: + # FancyLogger.get_logger().error( + # f"Found the string `CBlock` in what should've been a stringified context: {ctx_as_conversation}" + # ) + + conversation: list[dict] = [] + system_prompt = model_options.get(ModelOption.SYSTEM_PROMPT, "") + if system_prompt != "": + conversation.append({"role": "system", "content": system_prompt}) + + # TODO: JAL. Fix where this function lives. + conversation.extend( + [OpenAIBackend.message_to_openai_message(m) for m in ctx_as_message_list] + ) + + # handle custom system prompts. It's important that we do this before the _parse_and_**clean**_model_options step. + # system_prompt = model_options.get(ModelOption.SYSTEM_PROMPT, None) + # if system_prompt is not None: + # system_msg: dict[str, str] = { + # "role": "system", + # "content": system_prompt, + # } + # ctx_as_conversation.insert(0, system_msg) + + seed = model_options.get(ModelOption.SEED, None) + if seed is not None: + set_seed(seed) + + # TODO: JAL. Operates over ChatCompletion Requests... + # TODO: Check that the base_model_name matches the backend's model? + intrinsic_config = action.config + if intrinsic_config is None: + # If the intrinsic wasn't initialized with a config, grab one here based + # off the backend's model. + intrinsic_config_file = granite_common.intrinsics.util.obtain_io_yaml( + action.intrinsic_name, + self.model_id, # TODO: JAL. We need to a specific string here if model id isn't one...; probably the huggingface id + ) + intrinsic_config = granite_common.intrinsics.util.make_config_dict( + config_file=intrinsic_config_file + ) + intrinsic_config = cast( + dict, intrinsic_config + ) # Can remove if util function gets exported properly. + + # Check if there's a model name associated with this intrinsic (either user specified + # or specified in the config). + model_name = action.model_name + if model_name is None: + model_name = intrinsic_config["model"] + # TODO: JAL. model_name is only really used for routing the request. See what we do with + # vllm / openai with aloras. If the model name is the same and we just activate + # the adapters, that's fine; we can leave the model name alone... + # - with openai / vllm, you actually do use the lora adapter name... + # - with openai / vllm, you use the alora name... + # I think this also means no generation lock is required for OpenAI; only huggingface + + # TODO: JAL. Use the better matching function. Going by name alone right now. + # Also move this to be an early exit. + # TODO: JAL. Check for LORAs as well once that interface is done. + adapter = self.get_alora(action.intrinsic_name + "_" + "alora") + if adapter is None: + raise ValueError( + f"no alora / lora for processing intrinsic: {action.intrinsic_name}" + ) + + intrinsic_config["response_format"] = json.dumps( + intrinsic_config["response_format"] + ) + rewriter_config = deepcopy(intrinsic_config) + result_processor_config = deepcopy(intrinsic_config) + + # TODO: JAL. The name passed to the intrinsics rewriter should be the alora's or the intrinsics? theoretically + # the alora's should always be the intrinsic's given the matching func. + rewriter = granite_common.IntrinsicsRewriter( + config_dict=rewriter_config, model_name=adapter.name + ) + result_processor = granite_common.IntrinsicsResultProcessor( + config_dict=result_processor_config + ) + + # Convert our conversation into a proper chat completions dict. + # [{role: user, content: Hello}, {...}] -> {messages: [{role:user,...}, ...], model:..., ...} + request_json: dict = {"messages": conversation} + rewritten = rewriter.transform(request_json, **action.intrinsic_kwargs) + + # TODO: JAL. Will need a lock here... + self._model.set_adapter(adapter.qualified_name) + generate_input, other_input = ( + granite_common.util.chat_completion_request_to_transformers_inputs( + rewritten, self._tokenizer, self._model + ) + ) + + # TODO: Will need to break this piece out to support caching... + chat_response = granite_common.util.generate_with_transformers( + self._tokenizer, self._model, generate_input, other_input + ) + processed_chat_completion = result_processor.transform(chat_response, rewritten) + + # TODO: JAL. Put into ModelOutputThunk, parse, etc... + mot = ModelOutputThunk(processed_chat_completion.choices[0].message.content) + mot._generate_log = GenerateLog() + + # TODO: JAL. Move this deactivation elsewhere. + self._model.disable_adapters() + return mot + def _generate_from_context_standard( self, action: Component | CBlock, @@ -696,6 +846,20 @@ def _extract_model_tool_requests( return None # region ALora loading, unloading, and utility functions. + def add_adapter(self, adapter: LocalHFAdapter): + base_model_name = self._hf_model_id.split("/")[1] + path = adapter.get_local_hf_path(base_model_name) + + if self.get_alora(adapter.qualified_name) is not None: + FancyLogger.get_logger().warning( + f"Client code attempted to add {adapter.name} but {adapter.name} was already added to {self.__class__}. The backend is refusing to do this, because adapter loading is not idempotent." + ) + return None + + self._model.load_adapter(path, adapter.qualified_name) + self._model.disable_adapters() + self._aloras[adapter.qualified_name] = adapter + def add_alora(self, alora: HFAlora): """Loads an ALora for this backend. diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 39b026a87..767274bab 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -7,19 +7,19 @@ import inspect import json from collections.abc import Callable, Coroutine -from enum import Enum -from typing import TYPE_CHECKING, Any -from urllib.parse import urlparse +from copy import deepcopy +from typing import TYPE_CHECKING, Any, cast +import granite_common import openai import requests -from huggingface_hub import snapshot_download from openai.types.chat import ChatCompletion from openai.types.chat.chat_completion_chunk import ChatCompletionChunk from openai.types.completion import Completion import mellea.backends.model_ids as model_ids from mellea.backends import BaseModelSubclass +from mellea.backends.adapters.adapter import OpenAIAdapter from mellea.backends.aloras import Alora, AloraBackendMixin from mellea.backends.formatter import Formatter, FormatterBackend, TemplateFormatter from mellea.backends.model_ids import ModelIdentifier @@ -28,7 +28,7 @@ add_tools_from_model_options, convert_tools_to_json, ) -from mellea.backends.types import ModelOption +from mellea.backends.types import ModelOption, _server_type, _ServerType from mellea.helpers.async_helpers import ( ClientCache, get_current_event_loop, @@ -48,6 +48,7 @@ ModelOutputThunk, ) from mellea.stdlib.chat import Message +from mellea.stdlib.intrinsics.intrinsic import Intrinsic from mellea.stdlib.requirement import ALoraRequirement, LLMaJRequirement, Requirement if TYPE_CHECKING: @@ -58,24 +59,6 @@ format: None = None # typing this variable in order to shadow the global format function and ensure mypy checks for errors -class _ServerType(Enum): - LOCALHOST = 1 - OPENAI = 2 - - -def _server_type(url: str) -> _ServerType | None: - try: - parsed = urlparse(url) - hostname = parsed.hostname - if hostname in ("localhost", "127.0.0.1", "::1"): - return _ServerType.LOCALHOST - elif hostname == "api.openai.com": - return _ServerType.OPENAI - except Exception as e: - print(f"Error parsing URL: {e}") - return None - - class OpenAIBackend(FormatterBackend, AloraBackendMixin): """A generic OpenAI compatible backend.""" @@ -170,6 +153,8 @@ def __init__( else: self._api_key = api_key + self._server_type = _server_type(self._base_url) + self._openai_client_kwargs = self.filter_openai_client_kwargs(**kwargs) self._client = openai.OpenAI( # type: ignore @@ -337,6 +322,11 @@ def generate_from_chat_context( action, ctx, _format=_format, model_options=model_options ) + elif isinstance(action, Intrinsic): + return self._generate_from_intrinsic( + action, ctx, _format=format, model_options=model_options + ) + return self._generate_from_chat_context_standard( action, ctx, @@ -396,6 +386,177 @@ def _generate_from_chat_context_alora( return alora_output + # TODO: JAL. Do we need a mixin that radios something like this is supported? yes, we need to signal that you can add these adapters? + def _generate_from_intrinsic( + self, + action: Intrinsic, + ctx: Context, + *, + _format: type[BaseModelSubclass] | None = None, # TODO: JAL. Remove this param? + model_options: dict | None = None, + tool_calls: bool = False, # TODO: JAL. Remove this param? + ) -> ModelOutputThunk: + model_opts = self._simplify_and_merge( + model_options, is_chat_context=ctx.is_chat_context + ) # TODO: JAL. Might just ignore passed in model_opts. + + linearized_context = ctx.view_for_generation() + assert linearized_context is not None, ( + "Cannot generate from a non-linear context in a FormatterBackend." + ) # TODO: JAL. Need to handle simple contexts... + + # Convert our linearized context into a sequence of chat messages. Template formatters have a standard way of doing this. + messages: list[Message] = self.formatter.to_chat_messages(linearized_context) + + conversation: list[dict] = [] + + # TODO: JAL. Need to handle system prompts? Confirm what granite common does. + system_prompt = model_opts.get(ModelOption.SYSTEM_PROMPT, "") + if system_prompt != "": + conversation.append({"role": "system", "content": system_prompt}) + conversation.extend([self.message_to_openai_message(m) for m in messages]) + + # TODO: Check that the base_model_name matches the backend's model? + intrinsic_config = action.config + if intrinsic_config is None: + # If the intrinsic wasn't initialized with a config, grab one here based + # off the backend's model. + intrinsic_config_file = granite_common.intrinsics.util.obtain_io_yaml( + action.intrinsic_name, + self.model_id, # TODO: JAL. We need to a specific string here if model id isn't one...; probably the huggingface id + ) + intrinsic_config = granite_common.intrinsics.util.make_config_dict( + config_file=intrinsic_config_file + ) + intrinsic_config = cast( + dict, intrinsic_config + ) # Can remove if util function gets exported properly. + + # Check if there's a model name associated with this intrinsic (either user specified + # or specified in the config). + model_name = action.model_name + if model_name is None: + model_name = intrinsic_config["model"] + # TODO: JAL. model_name is only really used for routing the request. See what we do with + # vllm / openai with aloras. If the model name is the same and we just activate + # the adapters, that's fine; we can leave the model name alone... + # - with openai / vllm, you actually do use the lora adapter name... + # - with openai / vllm, you use the alora name... + # I think this also means no generation lock is required for OpenAI; only huggingface + + # TODO: JAL. Use the better matching function. Going by name alone right now. + # Also move this to be an early exit. + # TODO: JAL. Check for LORAs as well once that interface is done. + # TODO: JAL. This will have to have some sort of intrinsic compat field? + adapter = self.get_alora(action.intrinsic_name + "_" + "alora") + if adapter is None: + raise ValueError( + f"no alora / lora for processing intrinsic: {action.intrinsic_name}" + ) + + def adapter_intrinsic_match( + intrinsic_name: str, + model_name: str | None, + adapter_name: str, + adapter_path: str | None, + ) -> bool: + """Checks if a given intrinsic matches/corresponds to a given adapter. + + If no model_name is provided, checks that the intrinsic_name and adapter_name match. + Otherwise, checks that the model_name and the adapter_path match. + + Args: + intrinsic_name: the name of the intrinsic, like "answerability" + model_name: the name of the model specified by the intrinsic, typically a huggingface model id + adapter_name: the name of the adapter, like "answerability" or "constraint" + adapter_path: the huggingface path or model id + """ + if model_name is None: + # If no model_name is provided, the only check we perform is if we have a corresponding adapter to load. + return intrinsic_name == adapter_name + + # If the model_name and adapter_path match, we assume their names don't matter. + return model_name == adapter_path + + # TODO: JAL. Check that this backend has an adapter suitable for intrinsic. + # Checks that must be done: + # - if no model_name, check for a matching lora / alora name + # - if model_name, see if we have a matching lora / alora model + # - model_name might be stored as a part of the alora's path? need to investigate further. + # Both of these checks need to also check if the Intrinsic is alora / lora compatible + + # Necessary for the rewriter and result processor. Response format must be a string, even though we are using + # granite_common to parse the dict from a provided file. The rewriter also modifies the dict passed in, + # so we have to pass separate deep copies to the input and result processors. + intrinsic_config["response_format"] = json.dumps( + intrinsic_config["response_format"] + ) + rewriter_config = deepcopy(intrinsic_config) + result_processor_config = deepcopy(intrinsic_config) + + # TODO: JAL. The name passed to the intrinsics rewriter should be the alora's or the intrinsics? theoretically + # the alora's should always be the intrinsic's given the matching func. + rewriter = granite_common.IntrinsicsRewriter( + config_dict=rewriter_config, model_name=adapter.name + ) + result_processor = granite_common.IntrinsicsResultProcessor( + config_dict=result_processor_config + ) + + # Convert our conversation into a proper chat completions dict. + # [{role: user, content: Hello}, {...}] -> {messages: [{role:user,...}, ...], model:..., ...} + request_json: dict = {"messages": conversation} + rewritten = rewriter.transform(request_json, **action.intrinsic_kwargs) + + # TODO: JAL. + # Steps: + # See if the intrinsic specified a model / base model + # if it did: compare that to the current model / currently loaded loras and aloras + # if it did not: check current loras and aloras for the intrinsic name + # If match found: + # send off the request + # if match not found: + # look up more info... + + # TODO: JAL. These next two steps seem intertwined. How do you know if the alora/lora/base_model is available? + # TODO: JAL. Activate the lora / alora...; or call it? need to see how we interface there... + # Check if the current model is one that supports the intrinsic; if so, send the request directly to the backend. + # Otherwise, check if alora is available. + # Otherwise, check if lora is available. + + # TODO: JAL. Get the yaml file / configs from the intrinsic; if not there, look it up using below function... + # TODO: JAL. Message granite_intrinsic folks about util not being exported + # TODO: JAL. base_model_name should be done at the intrinsic level? and then checked against the + # current backend / loaded aloras / loaded loras + # if the intrinsic specifies a base_model_name, use that / do the comparison; otherwise, + # just check for alora / lora and be good + # in this way, intrinsics are not linked to a specific model; we should be able to do things + # based purely off the intrinsic name not the model + + # TODO: JAL. Populate any remaining params needed... + # TODO: JAL. additional kwargs like requirement text should be a part of the intrinsic + + # TODO: Intrinsic must also radio if it needs to add an additional message. + # no; the transform takes care of this... + + # TODO: JAL. Need to handle caching...? or cache lookup? + # - the intrinsics should define which messages they change?... idk... think about this more... + # TODO: JAL. This needs to be made async with processing and post_processing. + # chat_response: Coroutine[ + # Any, Any, openai.AsyncStream[Completion] | Completion + # ] = self._async_client.chat.completions.create( + # **rewritten.model_dump() + # ) + # # TODO: Need to figure out if interface changes for aloras are needed... + chat_response = self._client.chat.completions.create(**rewritten.model_dump()) + + processed_chat_completion = result_processor.transform(chat_response, rewritten) + + # TODO: JAL. Put into ModelOutputThunk, parse, etc... + mot = ModelOutputThunk(processed_chat_completion.choices[0].message.content) + mot._generate_log = GenerateLog() + return mot + @staticmethod def message_to_openai_message(msg: Message): """Serializes a mellea Message object to the message format required by OpenAI compatible api providers.""" @@ -718,6 +879,45 @@ def _generate_from_raw( return results + def add_adapter(self, adapter: OpenAIAdapter): + base_model_name = self._hf_model_id.split("/")[1] + path = adapter.get_open_ai_path(base_model_name, server_type=self._server_type) + + if self.get_alora(adapter.qualified_name) is not None: + FancyLogger.get_logger().warning( + f"Client code attempted to add {adapter.name} but {adapter.name} was already added to {self.__class__}. The backend is refusing to do this, because adapter loading is not idempotent." + ) + return None + + url = f"{self._base_url}/load_lora_adapter" + response = requests.post( + url, + json={"lora_name": adapter.qualified_name, "lora_path": path}, + headers={"Content-Type": "application/json"}, + ) + + err: str | None = None + match response.status_code: + case 200: + FancyLogger.get_logger().info( + f"{url}: status {response.status_code} {response.text}" + ) + case 400: + if "has already been loaded." in str(response.content): + FancyLogger.get_logger().warning( + f"{url}: status {response.status_code} {response.text}" + ) + else: + err = f"{url}: status {response.status_code} {response.text}" + case _: + err = f"{url}: status {response.status_code} {response.text}" + + if err is not None: + FancyLogger.get_logger().error(err) + raise Exception(f"error adding adapter: {err}") + + self._aloras[adapter.qualified_name] = adapter + def add_alora(self, alora: "OpenAIAlora"): """Loads an ALora for this backend. @@ -739,7 +939,13 @@ def add_alora(self, alora: "OpenAIAlora"): "alora is supported only for locally running vllm instances" ) - snapshot_path = snapshot_download(alora.path) + # TODO: JAL. Make sure all hf model ids fit this way. + # base_model_name = self._hf_model_id.split("/")[1] + # TODO: JAL. change snapshot path to this...? + # snapshot_path = granite_common.intrinsics.util.obtain_lora( + # alora.name, base_model_name, alora=True + # ) + snapshot_path = f"/u/jakelorocco/eiger-user-folder/mellea-public/test/backends/test_openai_vllm/rag-intrinsics-lib/{alora.name}/alora/granite-3.3-8b-instruct" # https://docs.vllm.ai/en/stable/features/lora.html#using-api-endpoints # curl -X POST http://localhost:8000/v1/load_lora_adapter \ @@ -756,6 +962,10 @@ def add_alora(self, alora: "OpenAIAlora"): headers={"Content-Type": "application/json"}, ) + # TODO: Add a check here for the lora/alora already being loaded. + # TODO: See what happens if you try load the lora with the same name... + # TODO: If the alora isn't loaded, we should raise an error or at least not + # add it to the list of aloras. match response.status_code: case 200: FancyLogger.get_logger().info( diff --git a/mellea/backends/types.py b/mellea/backends/types.py index d7f0db128..3c4e21a70 100644 --- a/mellea/backends/types.py +++ b/mellea/backends/types.py @@ -1,6 +1,8 @@ """Useful type definitions for models, formatters, and backends.""" +from enum import Enum from typing import Any +from urllib.parse import urlparse from mellea.helpers.fancy_logger import FancyLogger @@ -109,3 +111,24 @@ def merge_model_options( for k, v in overwrite_opts.items(): new_options[k] = v return new_options + + +class _ServerType(Enum): + UNKNOWN = 0 + LOCALHOST = 1 + OPENAI = 2 + REMOTE_VLLM = 3 + """Must be set manually for now.""" + + +def _server_type(url: str) -> _ServerType: + try: + parsed = urlparse(url) + hostname = parsed.hostname + if hostname in ("localhost", "127.0.0.1", "::1", "0.0.0.0"): + return _ServerType.LOCALHOST + elif hostname == "api.openai.com": + return _ServerType.OPENAI + except Exception as e: + print(f"Error parsing URL: {e}") + return _ServerType.UNKNOWN diff --git a/mellea/stdlib/intrinsics/intrinsic.py b/mellea/stdlib/intrinsics/intrinsic.py new file mode 100644 index 000000000..f62d4e334 --- /dev/null +++ b/mellea/stdlib/intrinsics/intrinsic.py @@ -0,0 +1,100 @@ +import pathlib +from typing import cast + +import granite_common + +from mellea.stdlib.base import CBlock, Component, TemplateRepresentation + + +class Intrinsic(Component): + """A component for rewriting messages using intrinsics. + + Intrinsics are special components that transform a chat completion request. + These transformations typically take the form of: + - parameter changes (typically structured outputs) + - adding new messages to the chat + - editing existing messages + (- adding documents? TODO: JAL investigate if this happens or if the docs are always user provided / already there...) + """ + + def __init__( + self, + intrinsic_name: str, + config_file: str | pathlib.Path | None = None, + config_dict: dict | None = None, + base_model_name: str + | None = None, # TODO: JAL. allow this to take a model_id? if so, need to change equivalence logic elsewhere... + model_name: str | None = None, + intrinsic_kwargs: dict | None = None, + # compatibility: list[AdapterType] = [AdapterType.ALORA, AdapterType.LORA] # TODO: JAL. Change this into an enum or literal list like roles... + ) -> None: + # TODO: JAL. Allow specifying an alora / lora directly as well...? instead of just by name...? + # if so, need to change matching logic... + + self.intrinsic_name = intrinsic_name + + # The base_model_name is used for looking up config files if needed. + self.base_model_name = base_model_name + + # The model_name is the model actually used for the intrinsic. + # Mellea currently only supports this being adapters. Typically don't need + # to specify this. + self.model_name = model_name + + if intrinsic_kwargs is None: + intrinsic_kwargs = {} + # TODO: JAL. Document this. This is what subclasses should use to pass needed + # kwargs to transform call. + self.intrinsic_kwargs = intrinsic_kwargs + + # TODO: JAL. Potentially remove this config code here. Not necessarily a + # real reason to do this here except for being able to do it once + # and precaching special handling... + + # If any of the optional params are specified, attempt to set up the + # config for the intrinsic here. + config: dict | None = None + if config_file is not None or config_dict is not None: + config = granite_common.intrinsics.util.make_config_dict( + config_file=config_file, config_dict=config_dict + ) + config = cast( + dict, config + ) # Can remove if util function gets exported properly. + + if config is None and self.base_model_name is not None: + io_yaml_file = granite_common.intrinsics.util.obtain_io_yaml( + self.intrinsic_name, self.base_model_name + ) + config = granite_common.intrinsics.util.make_config_dict( + config_file=io_yaml_file + ) + config = cast( + dict, config + ) # Can remove if util function gets exported properly. + + self.config: dict | None = config + + def parts(self) -> list[Component | CBlock]: + """The set of all the constituent parts of the `Intrinsic`. + + Will need to be implemented by subclasses since not all intrinsics are output + as text / messages. TODO: JAL. + """ + raise NotImplementedError("parts isn't implemented by default") + + def format_for_llm(self) -> TemplateRepresentation | str: + """Formats the `Intrinsic` into a `TemplateRepresentation` or string. + + Returns: a `TemplateRepresentation` or string + TODO: JAL. see if the base intrinsic should implement this. + ideally, this would be the intrinsic's instruction directive with the args populated + however, the rewriter's transform function handles that... + """ + raise NotImplementedError("format_for_llm isn't implemented by default") + + def transform(self) -> None: + ... + # TODO: JAL. need to see if this function should be defined at the component level or if the backend will call this... + # there has to be some part of the intrinsic that defines the args / changes that take place; maybe that's done in granite common though... + # The steps required here are different per backend... need to see where we want to define this... diff --git a/pyproject.toml b/pyproject.toml index 87533270c..b463c5b4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,8 @@ dependencies = [ "huggingface-hub>=0.33.4", "pillow", "math_verify", # Needed for Majority Voting Sampling Strategies. - "rouge_score" # Needed for Majority Voting Sampling Strategies. + "rouge_score", # Needed for Majority Voting Sampling Strategies. + "granite_common", # Needed for Intrinsics. ] [project.scripts] @@ -60,7 +61,8 @@ hf = [ "alora==0.2.0", "datasets>=4.0.0", "outlines<1.0.0", - "peft>=0.16.0", + # "peft>=0.17.2", # This package can be re-enabled once peft 0.17.2 has been released and below commit is confirmed to be a part of it. + "peft @ git+https://github.com/huggingface/peft.git@293aea5df6db240856a77f89955d1a89ce38b50d", "transformers>=4.53.2", "trl==0.19.1", ] diff --git a/uv.lock b/uv.lock index 7d9d045f2..62794b0c5 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'darwin'", @@ -1244,6 +1244,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/61/d4b89fec821f72385526e1b9d9a3a0385dda4a72b206d28049e2c7cd39b8/gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77", size = 208168, upload-time = "2025-07-24T03:45:52.517Z" }, ] +[[package]] +name = "granite-common" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/9b/c0e846c0517c7581e63901f90f23a28aa6758ec2b686171a5f23ba73ae48/granite_common-0.3.2.tar.gz", hash = "sha256:f5bc850573700f160bab0ae921d5156a5fd4594ed6806ae82ddf8a6043aa4331", size = 273140, upload-time = "2025-10-24T19:29:51.449Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/54/cf965d50fe493f4fb8ab1d4dea371a5d9d74d8e0bfdf2bd17f41a3af973b/granite_common-0.3.2-py3-none-any.whl", hash = "sha256:45d5a99e264f9e009215daf039c85a1e1ea216983962bcff2c75b4fa4a815d87", size = 77490, upload-time = "2025-10-24T19:29:49.97Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -2377,6 +2390,7 @@ dependencies = [ { name = "ansicolors" }, { name = "click" }, { name = "fastapi" }, + { name = "granite-common" }, { name = "huggingface-hub" }, { name = "jinja2" }, { name = "json5" }, @@ -2460,6 +2474,7 @@ requires-dist = [ { name = "datasets", marker = "extra == 'hf'", specifier = ">=4.0.0" }, { name = "docling", marker = "extra == 'docling'", specifier = ">=2.45.0" }, { name = "fastapi" }, + { name = "granite-common" }, { name = "huggingface-hub", specifier = ">=0.33.4" }, { name = "ibm-watsonx-ai", marker = "extra == 'watsonx'", specifier = ">=1.3.31" }, { name = "jinja2" }, @@ -2471,7 +2486,7 @@ requires-dist = [ { name = "ollama", specifier = ">=0.5.1" }, { name = "openai" }, { name = "outlines", marker = "extra == 'hf'", specifier = "<1.0.0" }, - { name = "peft", marker = "extra == 'hf'", specifier = ">=0.16.0" }, + { name = "peft", marker = "extra == 'hf'", git = "https://github.com/huggingface/peft.git?rev=293aea5df6db240856a77f89955d1a89ce38b50d" }, { name = "pillow" }, { name = "pydantic" }, { name = "requests", specifier = ">=2.32.3" }, @@ -3495,8 +3510,8 @@ wheels = [ [[package]] name = "peft" -version = "0.17.1" -source = { registry = "https://pypi.org/simple" } +version = "0.17.2.dev0" +source = { git = "https://github.com/huggingface/peft.git?rev=293aea5df6db240856a77f89955d1a89ce38b50d#293aea5df6db240856a77f89955d1a89ce38b50d" } dependencies = [ { name = "accelerate" }, { name = "huggingface-hub" }, @@ -3510,10 +3525,6 @@ dependencies = [ { name = "tqdm" }, { name = "transformers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/70/b8/2e79377efaa1e5f0d70a497db7914ffd355846e760ffa2f7883ab0f600fb/peft-0.17.1.tar.gz", hash = "sha256:e6002b42517976c290b3b8bbb9829a33dd5d470676b2dec7cb4df8501b77eb9f", size = 568192, upload-time = "2025-08-21T09:25:22.703Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/fe/a2da1627aa9cb6310b6034598363bd26ac301c4a99d21f415b1b2855891e/peft-0.17.1-py3-none-any.whl", hash = "sha256:3d129d64def3d74779c32a080d2567e5f7b674e77d546e3585138216d903f99e", size = 504896, upload-time = "2025-08-21T09:25:18.974Z" }, -] [[package]] name = "pexpect" From f61efd5bdb98823335a1d5e3284c5510abd94e51 Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Sun, 26 Oct 2025 21:54:24 -0400 Subject: [PATCH 04/38] test: basic doc support for openai --- mellea/backends/adapters/adapter.py | 2 ++ mellea/backends/openai.py | 34 ++++++++++++++++++++++++----- mellea/stdlib/base.py | 8 +++++++ mellea/stdlib/chat.py | 4 ++++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 88914df19..0d9c6703d 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -50,6 +50,7 @@ def __init__(self, name: str, adapter_type: AdapterType): class OpenAIAdapter(Adapter): + # TODO: JAL. May need to add plain path here as well. @abc.abstractmethod def get_open_ai_path( self, @@ -68,6 +69,7 @@ def get_open_ai_path( class LocalHFAdapter(Adapter): + # TODO: JAL. May need to add plain path here as well. @abc.abstractmethod def get_local_hf_path(self, base_model_name: str) -> str: """Returns the path needed to load the adapter. diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 767274bab..27dd4c61c 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -43,6 +43,7 @@ CBlock, Component, Context, + Document, GenerateLog, GenerateType, ModelOutputThunk, @@ -415,6 +416,7 @@ def _generate_from_intrinsic( if system_prompt != "": conversation.append({"role": "system", "content": system_prompt}) conversation.extend([self.message_to_openai_message(m) for m in messages]) + docs = self.messages_to_docs(messages) # TODO: Check that the base_model_name matches the backend's model? intrinsic_config = action.config @@ -450,9 +452,12 @@ def _generate_from_intrinsic( # TODO: JAL. This will have to have some sort of intrinsic compat field? adapter = self.get_alora(action.intrinsic_name + "_" + "alora") if adapter is None: - raise ValueError( - f"no alora / lora for processing intrinsic: {action.intrinsic_name}" - ) + # TODO: JAL. Remove this nesting. + adapter = self.get_alora(action.intrinsic_name + "_" + "lora") + if adapter is None: + raise ValueError( + f"no alora / lora for processing intrinsic: {action.intrinsic_name}" + ) def adapter_intrinsic_match( intrinsic_name: str, @@ -497,7 +502,7 @@ def adapter_intrinsic_match( # TODO: JAL. The name passed to the intrinsics rewriter should be the alora's or the intrinsics? theoretically # the alora's should always be the intrinsic's given the matching func. rewriter = granite_common.IntrinsicsRewriter( - config_dict=rewriter_config, model_name=adapter.name + config_dict=rewriter_config, model_name=adapter.qualified_name ) result_processor = granite_common.IntrinsicsResultProcessor( config_dict=result_processor_config @@ -505,7 +510,10 @@ def adapter_intrinsic_match( # Convert our conversation into a proper chat completions dict. # [{role: user, content: Hello}, {...}] -> {messages: [{role:user,...}, ...], model:..., ...} - request_json: dict = {"messages": conversation} + request_json: dict = { + "messages": conversation, + "extra_body": {"documents": docs}, + } rewritten = rewriter.transform(request_json, **action.intrinsic_kwargs) # TODO: JAL. @@ -592,6 +600,22 @@ def message_to_openai_message(msg: Message): # ] # } + @staticmethod + def messages_to_docs(msgs: list[Message]) -> list[dict[str, str]]: + docs: list[Document] = [] + for message in msgs: + if message._docs is not None: + docs.extend(message._docs) + + # TODO: We can add doc_ids here for vllm. + json_docs: list[dict[str, str]] = [] + for doc in docs: + json_doc = {"text": doc.text} + if doc.title is not None: + json_doc["title"] = doc.title + json_docs.append(json_doc) + return json_docs + def _generate_from_chat_context_standard( self, action: Component | CBlock, diff --git a/mellea/stdlib/base.py b/mellea/stdlib/base.py index bf0c19546..fe565e691 100644 --- a/mellea/stdlib/base.py +++ b/mellea/stdlib/base.py @@ -114,6 +114,14 @@ def __repr__(self): return f"ImageBlock({self._value}, {self._meta.__repr__()})" +# TODO: JAL. Make this a CBlock? +# TODO: JAL. Add support for passing in docs as model options? +class Document: + def __init__(self, text: str, title: str | None = None): + self.text = text + self.title = title + + @runtime_checkable class Component(Protocol): """A `Component` is a composite data structure that is intended to be represented to an LLM.""" diff --git a/mellea/stdlib/chat.py b/mellea/stdlib/chat.py index 7f5bbb4aa..c4b445a77 100644 --- a/mellea/stdlib/chat.py +++ b/mellea/stdlib/chat.py @@ -8,6 +8,7 @@ CBlock, Component, Context, + Document, ImageBlock, ModelOutputThunk, ModelToolCall, @@ -26,6 +27,7 @@ def __init__( content: str, *, images: None | list[ImageBlock] = None, + documents: None | list[Document] = None, ): """Initializer for Chat messages. @@ -33,10 +35,12 @@ def __init__( role (str): The role that this message came from (e.g., user, assistant). content (str): The content of the message. images (list[ImageBlock]): The images associated with the message if any. + documents (list[Document]): documents associated with the message if any. """ self.role = role self.content = content self._images = images + self._docs = documents @property def images(self) -> None | list[str]: From 73774105f96180dcabd103b3c133175c53b7aa8d Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Tue, 28 Oct 2025 09:02:11 -0400 Subject: [PATCH 05/38] fix: move intrinsic config setup to adapters --- mellea/backends/adapters/adapter.py | 41 +++++++++++++- mellea/backends/huggingface.py | 34 ++++++------ mellea/backends/openai.py | 62 ++++++++++----------- mellea/stdlib/intrinsics/intrinsic.py | 80 +++++++-------------------- 4 files changed, 109 insertions(+), 108 deletions(-) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 0d9c6703d..197022a79 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -1,7 +1,9 @@ import abc +import pathlib from enum import Enum import granite_common +from litellm import cast from mellea.backends.types import _ServerType @@ -81,18 +83,55 @@ def get_local_hf_path(self, base_model_name: str) -> str: class RagIntrinsicAdapter(OpenAIAdapter, LocalHFAdapter): - def __init__(self, name: str, adapter_type: AdapterType = AdapterType.ALORA): + def __init__( + self, + name: str, + adapter_type: AdapterType = AdapterType.ALORA, + config_file: str | pathlib.Path | None = None, + config_dict: dict | None = None, + base_model_name: str | None = None, + ): """An adapter that can be added to either an `OpenAIBackend` or a `LocalHFBackend`. Most rag-lib-intrinsics support lora or alora adapter types. Args: name: name of the adapter; when referencing this adapter, use adapter.qualified_name adapter_type: enum describing what type of adapter it is (ie LORA / ALORA) + config_file: optional; file for defining the intrinsic / transformations + config_dict: optional; dict for defining the intrinsic / transformations + base_model_name: optional; if provided with no config_file/config_dict, will be used to lookup the granite_common config for this adapter """ assert adapter_type == AdapterType.ALORA or adapter_type == AdapterType.LORA, ( f"{adapter_type} not supported" ) super().__init__(name, adapter_type) + self.base_model_name = base_model_name + + # If any of the optional params are specified, attempt to set up the + # config for the intrinsic here. + config: dict | None = None + if config_file is not None or config_dict is not None: + config = granite_common.intrinsics.util.make_config_dict( + config_file=config_file, config_dict=config_dict + ) + config = cast( + dict, config + ) # Can remove if util function gets exported properly. + + if config is None and self.base_model_name is not None: + is_alora = True if self.adapter_type == AdapterType.ALORA else False + io_yaml_file = granite_common.intrinsics.util.obtain_io_yaml( + self.name, self.base_model_name, alora=is_alora + ) + config = granite_common.intrinsics.util.make_config_dict( + config_file=io_yaml_file + ) + config = cast( + dict, config + ) # Can remove if util function gets exported properly. + + self.config: dict | None = config + def get_open_ai_path( self, base_model_name: str, diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index c19ff1524..b4147ba82 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -33,7 +33,7 @@ from transformers.generation.utils import GenerateDecoderOnlyOutput from mellea.backends import BaseModelSubclass -from mellea.backends.adapters.adapter import LocalHFAdapter +from mellea.backends.adapters.adapter import LocalHFAdapter, RagIntrinsicAdapter from mellea.backends.aloras import Alora, AloraBackendMixin from mellea.backends.cache import Cache, SimpleLRUCache from mellea.backends.formatter import Formatter, FormatterBackend, TemplateFormatter @@ -311,12 +311,23 @@ def _generate_from_intrinsic( if seed is not None: set_seed(seed) - # TODO: JAL. Operates over ChatCompletion Requests... + # TODO: JAL. Use the better matching function. Going by name alone right now. + # Also move this to be an early exit. + # TODO: JAL. Check for LORAs as well once that interface is done. + adapter = self.get_alora(action.intrinsic_name + "_" + "alora") + if adapter is None: + adapter = self.get_alora(action.intrinsic_name + "_" + "lora") + if adapter is None: + raise ValueError( + f"no alora / lora for processing intrinsic: {action.intrinsic_name}" + ) + + # TODO: JAL. Fix this once get_alora is changed. + assert isinstance(adapter, RagIntrinsicAdapter) # TODO: Check that the base_model_name matches the backend's model? - intrinsic_config = action.config + intrinsic_config = adapter.config if intrinsic_config is None: - # If the intrinsic wasn't initialized with a config, grab one here based - # off the backend's model. + # If the adapter wasn't initialized with a config, grab one here based off the backend's model. intrinsic_config_file = granite_common.intrinsics.util.obtain_io_yaml( action.intrinsic_name, self.model_id, # TODO: JAL. We need to a specific string here if model id isn't one...; probably the huggingface id @@ -326,11 +337,11 @@ def _generate_from_intrinsic( ) intrinsic_config = cast( dict, intrinsic_config - ) # Can remove if util function gets exported properly. + ) # TODO: Can remove if util function gets exported properly. # Check if there's a model name associated with this intrinsic (either user specified # or specified in the config). - model_name = action.model_name + model_name = adapter.base_model_name if model_name is None: model_name = intrinsic_config["model"] # TODO: JAL. model_name is only really used for routing the request. See what we do with @@ -340,15 +351,6 @@ def _generate_from_intrinsic( # - with openai / vllm, you use the alora name... # I think this also means no generation lock is required for OpenAI; only huggingface - # TODO: JAL. Use the better matching function. Going by name alone right now. - # Also move this to be an early exit. - # TODO: JAL. Check for LORAs as well once that interface is done. - adapter = self.get_alora(action.intrinsic_name + "_" + "alora") - if adapter is None: - raise ValueError( - f"no alora / lora for processing intrinsic: {action.intrinsic_name}" - ) - intrinsic_config["response_format"] = json.dumps( intrinsic_config["response_format"] ) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 27dd4c61c..a6bda6c01 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -19,7 +19,7 @@ import mellea.backends.model_ids as model_ids from mellea.backends import BaseModelSubclass -from mellea.backends.adapters.adapter import OpenAIAdapter +from mellea.backends.adapters.adapter import OpenAIAdapter, RagIntrinsicAdapter from mellea.backends.aloras import Alora, AloraBackendMixin from mellea.backends.formatter import Formatter, FormatterBackend, TemplateFormatter from mellea.backends.model_ids import ModelIdentifier @@ -387,7 +387,6 @@ def _generate_from_chat_context_alora( return alora_output - # TODO: JAL. Do we need a mixin that radios something like this is supported? yes, we need to signal that you can add these adapters? def _generate_from_intrinsic( self, action: Intrinsic, @@ -404,7 +403,7 @@ def _generate_from_intrinsic( linearized_context = ctx.view_for_generation() assert linearized_context is not None, ( "Cannot generate from a non-linear context in a FormatterBackend." - ) # TODO: JAL. Need to handle simple contexts... + ) # TODO: JAL. Need to handle simple contexts... test what happens if there's no messages in the context... # Convert our linearized context into a sequence of chat messages. Template formatters have a standard way of doing this. messages: list[Message] = self.formatter.to_chat_messages(linearized_context) @@ -418,34 +417,6 @@ def _generate_from_intrinsic( conversation.extend([self.message_to_openai_message(m) for m in messages]) docs = self.messages_to_docs(messages) - # TODO: Check that the base_model_name matches the backend's model? - intrinsic_config = action.config - if intrinsic_config is None: - # If the intrinsic wasn't initialized with a config, grab one here based - # off the backend's model. - intrinsic_config_file = granite_common.intrinsics.util.obtain_io_yaml( - action.intrinsic_name, - self.model_id, # TODO: JAL. We need to a specific string here if model id isn't one...; probably the huggingface id - ) - intrinsic_config = granite_common.intrinsics.util.make_config_dict( - config_file=intrinsic_config_file - ) - intrinsic_config = cast( - dict, intrinsic_config - ) # Can remove if util function gets exported properly. - - # Check if there's a model name associated with this intrinsic (either user specified - # or specified in the config). - model_name = action.model_name - if model_name is None: - model_name = intrinsic_config["model"] - # TODO: JAL. model_name is only really used for routing the request. See what we do with - # vllm / openai with aloras. If the model name is the same and we just activate - # the adapters, that's fine; we can leave the model name alone... - # - with openai / vllm, you actually do use the lora adapter name... - # - with openai / vllm, you use the alora name... - # I think this also means no generation lock is required for OpenAI; only huggingface - # TODO: JAL. Use the better matching function. Going by name alone right now. # Also move this to be an early exit. # TODO: JAL. Check for LORAs as well once that interface is done. @@ -490,6 +461,35 @@ def adapter_intrinsic_match( # - model_name might be stored as a part of the alora's path? need to investigate further. # Both of these checks need to also check if the Intrinsic is alora / lora compatible + # TODO: JAL. Fix this once get_alora is changed. + assert isinstance(adapter, RagIntrinsicAdapter) + # TODO: Check that the base_model_name matches the backend's model? + intrinsic_config = adapter.config + if intrinsic_config is None: + # If the adapter wasn't initialized with a config, grab one here based off the backend's model. + intrinsic_config_file = granite_common.intrinsics.util.obtain_io_yaml( + action.intrinsic_name, + self.model_id, # TODO: JAL. We need to a specific string here if model id isn't one...; probably the huggingface id + ) + intrinsic_config = granite_common.intrinsics.util.make_config_dict( + config_file=intrinsic_config_file + ) + intrinsic_config = cast( + dict, intrinsic_config + ) # TODO: Can remove if util function gets exported properly. + + # Check if there's a model name associated with this intrinsic (either user specified + # or specified in the config). + model_name = adapter.base_model_name + if model_name is None: + model_name = intrinsic_config["model"] + # TODO: JAL. model_name is only really used for routing the request. See what we do with + # vllm / openai with aloras. If the model name is the same and we just activate + # the adapters, that's fine; we can leave the model name alone... + # - with openai / vllm, you actually do use the lora adapter name... + # - with openai / vllm, you use the alora name... + # I think this also means no generation lock is required for OpenAI; only huggingface + # Necessary for the rewriter and result processor. Response format must be a string, even though we are using # granite_common to parse the dict from a provided file. The rewriter also modifies the dict passed in, # so we have to pass separate deep copies to the input and result processors. diff --git a/mellea/stdlib/intrinsics/intrinsic.py b/mellea/stdlib/intrinsics/intrinsic.py index f62d4e334..8e88f88ab 100644 --- a/mellea/stdlib/intrinsics/intrinsic.py +++ b/mellea/stdlib/intrinsics/intrinsic.py @@ -1,80 +1,46 @@ import pathlib +from copy import copy from typing import cast import granite_common +from mellea.backends.adapters.adapter import AdapterType from mellea.stdlib.base import CBlock, Component, TemplateRepresentation class Intrinsic(Component): - """A component for rewriting messages using intrinsics. - - Intrinsics are special components that transform a chat completion request. - These transformations typically take the form of: - - parameter changes (typically structured outputs) - - adding new messages to the chat - - editing existing messages - (- adding documents? TODO: JAL investigate if this happens or if the docs are always user provided / already there...) - """ + """A component representing an intrinsic.""" def __init__( self, intrinsic_name: str, - config_file: str | pathlib.Path | None = None, - config_dict: dict | None = None, - base_model_name: str - | None = None, # TODO: JAL. allow this to take a model_id? if so, need to change equivalence logic elsewhere... - model_name: str | None = None, intrinsic_kwargs: dict | None = None, - # compatibility: list[AdapterType] = [AdapterType.ALORA, AdapterType.LORA] # TODO: JAL. Change this into an enum or literal list like roles... + adapter_types: list[AdapterType] = [AdapterType.ALORA, AdapterType.LORA], ) -> None: - # TODO: JAL. Allow specifying an alora / lora directly as well...? instead of just by name...? - # if so, need to change matching logic... + """A component for rewriting messages using intrinsics. - self.intrinsic_name = intrinsic_name + Intrinsics are special components that transform a chat completion request. + These transformations typically take the form of: + - parameter changes (typically structured outputs) + - adding new messages to the chat + - editing existing messages + + An intrinsic component should correspond to a loaded adapter. - # The base_model_name is used for looking up config files if needed. - self.base_model_name = base_model_name + Args: + intrinsic_name: the name of the intrinsic; must match the adapter + intrinsic_kwargs: some intrinsics require kwargs when utilizing them; provide them here + adapter_types: list of adapter types that can be used for this intrinsic + """ + self.intrinsic_name = intrinsic_name - # The model_name is the model actually used for the intrinsic. - # Mellea currently only supports this being adapters. Typically don't need - # to specify this. - self.model_name = model_name + # Copy the list so that this intrinsic has its own list that can be modified independently. + self.adapter_types = copy(adapter_types) if intrinsic_kwargs is None: intrinsic_kwargs = {} - # TODO: JAL. Document this. This is what subclasses should use to pass needed - # kwargs to transform call. self.intrinsic_kwargs = intrinsic_kwargs - # TODO: JAL. Potentially remove this config code here. Not necessarily a - # real reason to do this here except for being able to do it once - # and precaching special handling... - - # If any of the optional params are specified, attempt to set up the - # config for the intrinsic here. - config: dict | None = None - if config_file is not None or config_dict is not None: - config = granite_common.intrinsics.util.make_config_dict( - config_file=config_file, config_dict=config_dict - ) - config = cast( - dict, config - ) # Can remove if util function gets exported properly. - - if config is None and self.base_model_name is not None: - io_yaml_file = granite_common.intrinsics.util.obtain_io_yaml( - self.intrinsic_name, self.base_model_name - ) - config = granite_common.intrinsics.util.make_config_dict( - config_file=io_yaml_file - ) - config = cast( - dict, config - ) # Can remove if util function gets exported properly. - - self.config: dict | None = config - def parts(self) -> list[Component | CBlock]: """The set of all the constituent parts of the `Intrinsic`. @@ -92,9 +58,3 @@ def format_for_llm(self) -> TemplateRepresentation | str: however, the rewriter's transform function handles that... """ raise NotImplementedError("format_for_llm isn't implemented by default") - - def transform(self) -> None: - ... - # TODO: JAL. need to see if this function should be defined at the component level or if the backend will call this... - # there has to be some part of the intrinsic that defines the args / changes that take place; maybe that's done in granite common though... - # The steps required here are different per backend... need to see where we want to define this... From 2d627ad176b1ac92ffb20597ea043a339f6724c7 Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Tue, 28 Oct 2025 13:00:21 -0400 Subject: [PATCH 06/38] test: push most recent changes --- mellea/backends/adapters/adapter.py | 25 ++++++ mellea/backends/openai.py | 131 +++++----------------------- 2 files changed, 45 insertions(+), 111 deletions(-) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 197022a79..3c8a8fa1c 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -173,3 +173,28 @@ def download_and_get_path(self, base_model_name: str) -> str: def get_path_on_remote(self, base_model_name: str, base_path: str) -> str: """Assumes the files have already been downloaded on the remote server.""" return f"./{base_path}/{self.name}/{self.adapter_type.value}/{base_model_name}" + + +def get_adapter_for_intrinsic( + intrinsic_name: str, + intrinsic_adapter_types: list[AdapterType], + available_adapters: dict[str, Adapter], +) -> Adapter | None: + """Finds an adapter from a dict of available adapters based on the intrinsic name and its allowed adapter types. + + Args: + intrinsic_name: the name of the intrinsic, like "answerability" + intrinsic_adapter_types: the adapter types allowed for this intrinsic, like ALORA / LORA + available_adapters: the available adapters to choose from; maps adapter.qualified_name to the Adapter + + Returns: + an Adapter if found; else None + """ + adapter = None + for adapter_type in intrinsic_adapter_types: + qualified_name = intrinsic_name + "_" + adapter_type.value + adapter = available_adapters.get(qualified_name, None) + if adapter is not None: + break + + return adapter diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index a6bda6c01..3d8853a3c 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -19,7 +19,7 @@ import mellea.backends.model_ids as model_ids from mellea.backends import BaseModelSubclass -from mellea.backends.adapters.adapter import OpenAIAdapter, RagIntrinsicAdapter +from mellea.backends.adapters.adapter import OpenAIAdapter, RagIntrinsicAdapter, get_adapter_for_intrinsic from mellea.backends.aloras import Alora, AloraBackendMixin from mellea.backends.formatter import Formatter, FormatterBackend, TemplateFormatter from mellea.backends.model_ids import ModelIdentifier @@ -325,7 +325,7 @@ def generate_from_chat_context( elif isinstance(action, Intrinsic): return self._generate_from_intrinsic( - action, ctx, _format=format, model_options=model_options + action, ctx, model_options=model_options ) return self._generate_from_chat_context_standard( @@ -392,9 +392,7 @@ def _generate_from_intrinsic( action: Intrinsic, ctx: Context, *, - _format: type[BaseModelSubclass] | None = None, # TODO: JAL. Remove this param? model_options: dict | None = None, - tool_calls: bool = False, # TODO: JAL. Remove this param? ) -> ModelOutputThunk: model_opts = self._simplify_and_merge( model_options, is_chat_context=ctx.is_chat_context @@ -403,7 +401,7 @@ def _generate_from_intrinsic( linearized_context = ctx.view_for_generation() assert linearized_context is not None, ( "Cannot generate from a non-linear context in a FormatterBackend." - ) # TODO: JAL. Need to handle simple contexts... test what happens if there's no messages in the context... + ) # TODO: JAL. Log a warning if this is empty?... # Convert our linearized context into a sequence of chat messages. Template formatters have a standard way of doing this. messages: list[Message] = self.formatter.to_chat_messages(linearized_context) @@ -415,61 +413,24 @@ def _generate_from_intrinsic( if system_prompt != "": conversation.append({"role": "system", "content": system_prompt}) conversation.extend([self.message_to_openai_message(m) for m in messages]) - docs = self.messages_to_docs(messages) + docs = self.messages_to_docs(messages) # TODO: JAL. Improve docs interface. - # TODO: JAL. Use the better matching function. Going by name alone right now. - # Also move this to be an early exit. - # TODO: JAL. Check for LORAs as well once that interface is done. - # TODO: JAL. This will have to have some sort of intrinsic compat field? - adapter = self.get_alora(action.intrinsic_name + "_" + "alora") + adapter = get_adapter_for_intrinsic(action.intrinsic_name, action.adapter_types, self._aloras) if adapter is None: - # TODO: JAL. Remove this nesting. - adapter = self.get_alora(action.intrinsic_name + "_" + "lora") - if adapter is None: - raise ValueError( - f"no alora / lora for processing intrinsic: {action.intrinsic_name}" - ) + raise ValueError( + f"backend ({self}) has no adapter for processing intrinsic: {action.intrinsic_name}" + ) - def adapter_intrinsic_match( - intrinsic_name: str, - model_name: str | None, - adapter_name: str, - adapter_path: str | None, - ) -> bool: - """Checks if a given intrinsic matches/corresponds to a given adapter. - - If no model_name is provided, checks that the intrinsic_name and adapter_name match. - Otherwise, checks that the model_name and the adapter_path match. - - Args: - intrinsic_name: the name of the intrinsic, like "answerability" - model_name: the name of the model specified by the intrinsic, typically a huggingface model id - adapter_name: the name of the adapter, like "answerability" or "constraint" - adapter_path: the huggingface path or model id - """ - if model_name is None: - # If no model_name is provided, the only check we perform is if we have a corresponding adapter to load. - return intrinsic_name == adapter_name - - # If the model_name and adapter_path match, we assume their names don't matter. - return model_name == adapter_path - - # TODO: JAL. Check that this backend has an adapter suitable for intrinsic. - # Checks that must be done: - # - if no model_name, check for a matching lora / alora name - # - if model_name, see if we have a matching lora / alora model - # - model_name might be stored as a part of the alora's path? need to investigate further. - # Both of these checks need to also check if the Intrinsic is alora / lora compatible - - # TODO: JAL. Fix this once get_alora is changed. + # TODO: Code below this point is mostly specific to RagIntrinsics (and granite_common). + # It should be refactored into a specific adapter.transform() function. assert isinstance(adapter, RagIntrinsicAdapter) - # TODO: Check that the base_model_name matches the backend's model? + intrinsic_config = adapter.config if intrinsic_config is None: # If the adapter wasn't initialized with a config, grab one here based off the backend's model. intrinsic_config_file = granite_common.intrinsics.util.obtain_io_yaml( action.intrinsic_name, - self.model_id, # TODO: JAL. We need to a specific string here if model id isn't one...; probably the huggingface id + self._hf_model_id.split("/")[-1], ) intrinsic_config = granite_common.intrinsics.util.make_config_dict( config_file=intrinsic_config_file @@ -478,34 +439,11 @@ def adapter_intrinsic_match( dict, intrinsic_config ) # TODO: Can remove if util function gets exported properly. - # Check if there's a model name associated with this intrinsic (either user specified - # or specified in the config). - model_name = adapter.base_model_name - if model_name is None: - model_name = intrinsic_config["model"] - # TODO: JAL. model_name is only really used for routing the request. See what we do with - # vllm / openai with aloras. If the model name is the same and we just activate - # the adapters, that's fine; we can leave the model name alone... - # - with openai / vllm, you actually do use the lora adapter name... - # - with openai / vllm, you use the alora name... - # I think this also means no generation lock is required for OpenAI; only huggingface - - # Necessary for the rewriter and result processor. Response format must be a string, even though we are using - # granite_common to parse the dict from a provided file. The rewriter also modifies the dict passed in, - # so we have to pass separate deep copies to the input and result processors. - intrinsic_config["response_format"] = json.dumps( - intrinsic_config["response_format"] - ) - rewriter_config = deepcopy(intrinsic_config) - result_processor_config = deepcopy(intrinsic_config) - - # TODO: JAL. The name passed to the intrinsics rewriter should be the alora's or the intrinsics? theoretically - # the alora's should always be the intrinsic's given the matching func. rewriter = granite_common.IntrinsicsRewriter( - config_dict=rewriter_config, model_name=adapter.qualified_name + config_dict=intrinsic_config, model_name=adapter.qualified_name ) result_processor = granite_common.IntrinsicsResultProcessor( - config_dict=result_processor_config + config_dict=intrinsic_config ) # Convert our conversation into a proper chat completions dict. @@ -514,48 +452,19 @@ def adapter_intrinsic_match( "messages": conversation, "extra_body": {"documents": docs}, } + rewritten = rewriter.transform(request_json, **action.intrinsic_kwargs) + # TODO: JAL. Move this comment to hugging face. + # TODO: Handle caching here. Need to see if granite_common gives us any indication + # of what messages have changed. We will also have to support caching at a + # Component / message level. - # TODO: JAL. - # Steps: - # See if the intrinsic specified a model / base model - # if it did: compare that to the current model / currently loaded loras and aloras - # if it did not: check current loras and aloras for the intrinsic name - # If match found: - # send off the request - # if match not found: - # look up more info... - - # TODO: JAL. These next two steps seem intertwined. How do you know if the alora/lora/base_model is available? - # TODO: JAL. Activate the lora / alora...; or call it? need to see how we interface there... - # Check if the current model is one that supports the intrinsic; if so, send the request directly to the backend. - # Otherwise, check if alora is available. - # Otherwise, check if lora is available. - - # TODO: JAL. Get the yaml file / configs from the intrinsic; if not there, look it up using below function... - # TODO: JAL. Message granite_intrinsic folks about util not being exported - # TODO: JAL. base_model_name should be done at the intrinsic level? and then checked against the - # current backend / loaded aloras / loaded loras - # if the intrinsic specifies a base_model_name, use that / do the comparison; otherwise, - # just check for alora / lora and be good - # in this way, intrinsics are not linked to a specific model; we should be able to do things - # based purely off the intrinsic name not the model - - # TODO: JAL. Populate any remaining params needed... - # TODO: JAL. additional kwargs like requirement text should be a part of the intrinsic - - # TODO: Intrinsic must also radio if it needs to add an additional message. - # no; the transform takes care of this... - - # TODO: JAL. Need to handle caching...? or cache lookup? - # - the intrinsics should define which messages they change?... idk... think about this more... # TODO: JAL. This needs to be made async with processing and post_processing. # chat_response: Coroutine[ # Any, Any, openai.AsyncStream[Completion] | Completion # ] = self._async_client.chat.completions.create( # **rewritten.model_dump() # ) - # # TODO: Need to figure out if interface changes for aloras are needed... chat_response = self._client.chat.completions.create(**rewritten.model_dump()) processed_chat_completion = result_processor.transform(chat_response, rewritten) @@ -904,7 +813,7 @@ def _generate_from_raw( return results def add_adapter(self, adapter: OpenAIAdapter): - base_model_name = self._hf_model_id.split("/")[1] + base_model_name = self._hf_model_id.split("/")[-1] path = adapter.get_open_ai_path(base_model_name, server_type=self._server_type) if self.get_alora(adapter.qualified_name) is not None: From 610ff74b465dd74ca12d7edb0a960f3a6085f031 Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Tue, 28 Oct 2025 20:58:44 -0400 Subject: [PATCH 07/38] test: split adapter add and load functions --- mellea/backends/adapters/adapter.py | 5 +- mellea/backends/huggingface.py | 4 +- mellea/backends/openai.py | 91 ++++++++++++++++++++++++----- 3 files changed, 82 insertions(+), 18 deletions(-) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 3c8a8fa1c..cc286dc99 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -26,6 +26,9 @@ def __init__(self, name: str, adapter_type: AdapterType): self.qualified_name = name + "_" + adapter_type.value """the name of the adapter to use when loading / looking it up""" + self.path: str | None = None + """set when the adapter is added to a backend""" + # @abc.abstractmethod # def add_adapter(self, backend: Backend, *args, **kwargs): # """Adds an adapter.""" @@ -82,7 +85,7 @@ def get_local_hf_path(self, base_model_name: str) -> str: ... -class RagIntrinsicAdapter(OpenAIAdapter, LocalHFAdapter): +class GraniteCommonAdapter(OpenAIAdapter, LocalHFAdapter): def __init__( self, name: str, diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index b4147ba82..4251ca38e 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -33,7 +33,7 @@ from transformers.generation.utils import GenerateDecoderOnlyOutput from mellea.backends import BaseModelSubclass -from mellea.backends.adapters.adapter import LocalHFAdapter, RagIntrinsicAdapter +from mellea.backends.adapters.adapter import GraniteCommonAdapter, LocalHFAdapter from mellea.backends.aloras import Alora, AloraBackendMixin from mellea.backends.cache import Cache, SimpleLRUCache from mellea.backends.formatter import Formatter, FormatterBackend, TemplateFormatter @@ -323,7 +323,7 @@ def _generate_from_intrinsic( ) # TODO: JAL. Fix this once get_alora is changed. - assert isinstance(adapter, RagIntrinsicAdapter) + assert isinstance(adapter, GraniteCommonAdapter) # TODO: Check that the base_model_name matches the backend's model? intrinsic_config = adapter.config if intrinsic_config is None: diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 3d8853a3c..7517d8c3a 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -19,7 +19,11 @@ import mellea.backends.model_ids as model_ids from mellea.backends import BaseModelSubclass -from mellea.backends.adapters.adapter import OpenAIAdapter, RagIntrinsicAdapter, get_adapter_for_intrinsic +from mellea.backends.adapters.adapter import ( + GraniteCommonAdapter, + OpenAIAdapter, + get_adapter_for_intrinsic, +) from mellea.backends.aloras import Alora, AloraBackendMixin from mellea.backends.formatter import Formatter, FormatterBackend, TemplateFormatter from mellea.backends.model_ids import ModelIdentifier @@ -388,11 +392,7 @@ def _generate_from_chat_context_alora( return alora_output def _generate_from_intrinsic( - self, - action: Intrinsic, - ctx: Context, - *, - model_options: dict | None = None, + self, action: Intrinsic, ctx: Context, *, model_options: dict | None = None ) -> ModelOutputThunk: model_opts = self._simplify_and_merge( model_options, is_chat_context=ctx.is_chat_context @@ -401,7 +401,7 @@ def _generate_from_intrinsic( linearized_context = ctx.view_for_generation() assert linearized_context is not None, ( "Cannot generate from a non-linear context in a FormatterBackend." - ) # TODO: JAL. Log a warning if this is empty?... + ) # TODO: JAL. Log a warning if this is empty?... # Convert our linearized context into a sequence of chat messages. Template formatters have a standard way of doing this. messages: list[Message] = self.formatter.to_chat_messages(linearized_context) @@ -413,9 +413,11 @@ def _generate_from_intrinsic( if system_prompt != "": conversation.append({"role": "system", "content": system_prompt}) conversation.extend([self.message_to_openai_message(m) for m in messages]) - docs = self.messages_to_docs(messages) # TODO: JAL. Improve docs interface. + docs = self.messages_to_docs(messages) # TODO: JAL. Improve docs interface. - adapter = get_adapter_for_intrinsic(action.intrinsic_name, action.adapter_types, self._aloras) + adapter = get_adapter_for_intrinsic( + action.intrinsic_name, action.adapter_types, self._aloras + ) if adapter is None: raise ValueError( f"backend ({self}) has no adapter for processing intrinsic: {action.intrinsic_name}" @@ -423,14 +425,13 @@ def _generate_from_intrinsic( # TODO: Code below this point is mostly specific to RagIntrinsics (and granite_common). # It should be refactored into a specific adapter.transform() function. - assert isinstance(adapter, RagIntrinsicAdapter) + assert isinstance(adapter, GraniteCommonAdapter) intrinsic_config = adapter.config if intrinsic_config is None: # If the adapter wasn't initialized with a config, grab one here based off the backend's model. intrinsic_config_file = granite_common.intrinsics.util.obtain_io_yaml( - action.intrinsic_name, - self._hf_model_id.split("/")[-1], + action.intrinsic_name, self._hf_model_id.split("/")[-1] ) intrinsic_config = granite_common.intrinsics.util.make_config_dict( config_file=intrinsic_config_file @@ -813,8 +814,17 @@ def _generate_from_raw( return results def add_adapter(self, adapter: OpenAIAdapter): + # Gets the path / downloads files... ie does setup. + # TODO: JAL. maybe just have a pointer to the backend as well... + if adapter.path is not None: + raise Exception( + f"adapter {adapter.name} has already been added to a backend" + ) + base_model_name = self._hf_model_id.split("/")[-1] - path = adapter.get_open_ai_path(base_model_name, server_type=self._server_type) + adapter.path = adapter.get_open_ai_path( + base_model_name, server_type=self._server_type + ) if self.get_alora(adapter.qualified_name) is not None: FancyLogger.get_logger().warning( @@ -822,10 +832,25 @@ def add_adapter(self, adapter: OpenAIAdapter): ) return None + # TODO: JAL. Add io yaml file download here? + # Need some sort of generic setup / add function here for each adapter. + + # TODO: JAL. Rename this internal field. Type should also be OpenAI adapter. + self._aloras[adapter.qualified_name] = adapter + + def load_adapter(self, adapter_qualified_name: str): + # Actually loads the adapter... + + adapter = self._aloras.get(adapter_qualified_name, None) + if adapter is None: + raise ValueError( + f"could not load adapter {adapter_qualified_name} for backend {self}: adapter was not previously added" + ) + url = f"{self._base_url}/load_lora_adapter" response = requests.post( url, - json={"lora_name": adapter.qualified_name, "lora_path": path}, + json={"lora_name": adapter_qualified_name, "lora_path": adapter.path}, headers={"Content-Type": "application/json"}, ) @@ -849,7 +874,43 @@ def add_adapter(self, adapter: OpenAIAdapter): FancyLogger.get_logger().error(err) raise Exception(f"error adding adapter: {err}") - self._aloras[adapter.qualified_name] = adapter + def unload_adapter(self, adapter_qualified_name: str): + # Unloads the adapter... + + adapter = self._aloras.get(adapter_qualified_name, None) + if adapter is None: + FancyLogger.get_logger().info( + f"could not unload adapter {adapter_qualified_name} for backend {self}: adapter was not previously added" + ) + return + + url = f"{self._base_url}/load_lora_adapter" + response = requests.post( + url, + json={"lora_name": adapter_qualified_name}, + headers={"Content-Type": "application/json"}, + ) + + # TODO: JAL. Test the response codes if already unloaded. + # err: str | None = None + # match response.status_code: + # case 200: + # FancyLogger.get_logger().info( + # f"{url}: status {response.status_code} {response.text}" + # ) + # case 400: + # if "has already been loaded." in str(response.content): + # FancyLogger.get_logger().warning( + # f"{url}: status {response.status_code} {response.text}" + # ) + # else: + # err = f"{url}: status {response.status_code} {response.text}" + # case _: + # err = f"{url}: status {response.status_code} {response.text}" + + # if err is not None: + # FancyLogger.get_logger().error(err) + # raise Exception(f"error adding adapter: {err}") def add_alora(self, alora: "OpenAIAlora"): """Loads an ALora for this backend. From e93b6292fa244685d79a941c3e8dc5237e975cfd Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Fri, 31 Oct 2025 17:05:01 -0400 Subject: [PATCH 08/38] fix: start cleaning up some code --- mellea/backends/adapters/adapter.py | 40 ++--- mellea/backends/aloras/__init__.py | 15 ++ mellea/backends/huggingface.py | 2 +- mellea/backends/openai.py | 219 ++++++++++++++------------ mellea/stdlib/base.py | 2 +- mellea/stdlib/intrinsics/intrinsic.py | 2 +- 6 files changed, 146 insertions(+), 134 deletions(-) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index cc286dc99..2621dd8f2 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -1,10 +1,12 @@ import abc import pathlib from enum import Enum +from typing import TypeVar import granite_common from litellm import cast +from mellea.backends import Backend from mellea.backends.types import _ServerType @@ -17,6 +19,8 @@ class Adapter(abc.ABC): def __init__(self, name: str, adapter_type: AdapterType): """An adapter that can be added to a backend. + Note: An adapter can only be added to a single backend. + Args: name: name of the adapter; when referencing this adapter, use adapter.qualified_name adapter_type: enum describing what type of adapter it is (ie LORA / ALORA) @@ -26,36 +30,14 @@ def __init__(self, name: str, adapter_type: AdapterType): self.qualified_name = name + "_" + adapter_type.value """the name of the adapter to use when loading / looking it up""" - self.path: str | None = None + self.backend: Backend | None = None """set when the adapter is added to a backend""" - # @abc.abstractmethod - # def add_adapter(self, backend: Backend, *args, **kwargs): - # """Adds an adapter.""" - - # def already_added(self, backend: Backend) -> bool: - # """Checks if a this adapter has already been loaded to a backend and if so, if it's the current backend. - - # Args: - # backend: the Backend to add the adapter to. - - # Returns: - # True if already added; False if not already added. - - # Raises: - # ValueError: if the adapter has already been added to a different backend. - # """ - # if self.backend is not None: - # if self.backend is not backend: - # raise ValueError(f"adapter ({self.name}, {self}) has already been added to a different backend ({self.backend})") - # else: - # # It's already been added to this backend. Do nothing. - # return True - # return False + self.path: str | None = None + """set when the adapter is added to a backend""" class OpenAIAdapter(Adapter): - # TODO: JAL. May need to add plain path here as well. @abc.abstractmethod def get_open_ai_path( self, @@ -74,7 +56,6 @@ def get_open_ai_path( class LocalHFAdapter(Adapter): - # TODO: JAL. May need to add plain path here as well. @abc.abstractmethod def get_local_hf_path(self, base_model_name: str) -> str: """Returns the path needed to load the adapter. @@ -178,11 +159,14 @@ def get_path_on_remote(self, base_model_name: str, base_path: str) -> str: return f"./{base_path}/{self.name}/{self.adapter_type.value}/{base_model_name}" +T = TypeVar("T") + + def get_adapter_for_intrinsic( intrinsic_name: str, intrinsic_adapter_types: list[AdapterType], - available_adapters: dict[str, Adapter], -) -> Adapter | None: + available_adapters: dict[str, T], +) -> T | None: """Finds an adapter from a dict of available adapters based on the intrinsic name and its allowed adapter types. Args: diff --git a/mellea/backends/aloras/__init__.py b/mellea/backends/aloras/__init__.py index ae7b37b2d..9f8cc3494 100644 --- a/mellea/backends/aloras/__init__.py +++ b/mellea/backends/aloras/__init__.py @@ -2,9 +2,11 @@ import abc +from mellea.backends.adapters.adapter import Adapter from mellea.stdlib.base import CBlock, ModelOutputThunk +# TODO: JAL. Remove this. class Alora(abc.ABC): """Activated LoRAs (Aloras)](https://arxiv.org/pdf/2504.12397) are are [low-rank adapters](https://arxiv.org/abs/2106.09685) that can reuse KV cache from their underlying model. @@ -55,3 +57,16 @@ def get_alora(self, alora_name: str) -> Alora | None: def get_aloras(self) -> list[Alora]: """Returns a list of all loaded aLoRA adapters.""" ... + + +class AdapterMixin(abc.ABC): + """Mixin class for backends capable of utilizing adapters.""" + + def add_adapter(self, *args, **kwargs): + """Adds the given adapter to the backend. Must not have been added to a different backend.""" + + def load_adapter(self, adapter_qualified_name: str): + """Loads the given adapter for the backend. Must have previously been added.""" + + def unload_adapter(self, adapter_qualified_name: str): + """Unloads the given adapter from the backend.""" diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index 4251ca38e..baa7825e1 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -261,7 +261,7 @@ def _generate_from_intrinsic( _format: type[BaseModelSubclass] | None = None, # TODO: JAL. Remove this param? model_options: dict[str, Any], tool_calls: bool = False, # TODO: JAL. Remove this param? - ) -> ModelOutputThunk: + ) -> ModelOutputThunk: # TODO: JAL. Return context here as well? Else treat like Aloras and add the context higher up. if not ctx.is_chat_context: raise Exception("Does not yet support non-chat contexts.") diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 7517d8c3a..028631b7e 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -24,7 +24,7 @@ OpenAIAdapter, get_adapter_for_intrinsic, ) -from mellea.backends.aloras import Alora, AloraBackendMixin +from mellea.backends.aloras import AdapterMixin, Alora, AloraBackendMixin from mellea.backends.formatter import Formatter, FormatterBackend, TemplateFormatter from mellea.backends.model_ids import ModelIdentifier from mellea.backends.tools import ( @@ -64,7 +64,7 @@ format: None = None # typing this variable in order to shadow the global format function and ensure mypy checks for errors -class OpenAIBackend(FormatterBackend, AloraBackendMixin): +class OpenAIBackend(FormatterBackend, AdapterMixin): """A generic OpenAI compatible backend.""" def __init__( @@ -172,7 +172,8 @@ def __init__( _ = self._async_client # ALoras that have been loaded for this model. - self._aloras: dict[str, OpenAIAlora] = {} + self._added_adapters: dict[str, OpenAIAdapter] = {} + self._loaded_adapters: dict[str, OpenAIAdapter] = {} @property def _async_client(self) -> openai.AsyncOpenAI: @@ -396,7 +397,11 @@ def _generate_from_intrinsic( ) -> ModelOutputThunk: model_opts = self._simplify_and_merge( model_options, is_chat_context=ctx.is_chat_context - ) # TODO: JAL. Might just ignore passed in model_opts. + ) + if len(model_opts.items()) > 0: + FancyLogger.get_logger().info( + "passing in model options when generating with an adapter; some model options may be overwritten / ignored" + ) linearized_context = ctx.view_for_generation() assert linearized_context is not None, ( @@ -416,7 +421,7 @@ def _generate_from_intrinsic( docs = self.messages_to_docs(messages) # TODO: JAL. Improve docs interface. adapter = get_adapter_for_intrinsic( - action.intrinsic_name, action.adapter_types, self._aloras + action.intrinsic_name, action.adapter_types, self._added_adapters ) if adapter is None: raise ValueError( @@ -466,6 +471,7 @@ def _generate_from_intrinsic( # ] = self._async_client.chat.completions.create( # **rewritten.model_dump() # ) + self.load_adapter(adapter.qualified_name) chat_response = self._client.chat.completions.create(**rewritten.model_dump()) processed_chat_completion = result_processor.transform(chat_response, rewritten) @@ -815,33 +821,34 @@ def _generate_from_raw( def add_adapter(self, adapter: OpenAIAdapter): # Gets the path / downloads files... ie does setup. - # TODO: JAL. maybe just have a pointer to the backend as well... - if adapter.path is not None: - raise Exception( - f"adapter {adapter.name} has already been added to a backend" - ) + if adapter.backend is not None: + if adapter.backend is self: + FancyLogger.get_logger().warning( + f"attempted to add adapter {adapter.name} with type {adapter.adapter_type} to the same backend {adapter.backend}" + ) + return + else: + raise Exception( + f"adapter {adapter.name} with type {adapter.adapter_type} has already been added to backend {adapter.backend}" + ) base_model_name = self._hf_model_id.split("/")[-1] adapter.path = adapter.get_open_ai_path( base_model_name, server_type=self._server_type ) - if self.get_alora(adapter.qualified_name) is not None: + if self._added_adapters.get(adapter.qualified_name, None) is not None: FancyLogger.get_logger().warning( - f"Client code attempted to add {adapter.name} but {adapter.name} was already added to {self.__class__}. The backend is refusing to do this, because adapter loading is not idempotent." + f"Client code attempted to add {adapter.name} with type {adapter.adapter_type} but it was already added to {self.__class__}. This attempt to add the adapter will be ignored." ) return None - # TODO: JAL. Add io yaml file download here? - # Need some sort of generic setup / add function here for each adapter. - - # TODO: JAL. Rename this internal field. Type should also be OpenAI adapter. - self._aloras[adapter.qualified_name] = adapter + self._added_adapters[adapter.qualified_name] = adapter def load_adapter(self, adapter_qualified_name: str): # Actually loads the adapter... - adapter = self._aloras.get(adapter_qualified_name, None) + adapter = self._added_adapters.get(adapter_qualified_name, None) if adapter is None: raise ValueError( f"could not load adapter {adapter_qualified_name} for backend {self}: adapter was not previously added" @@ -872,116 +879,122 @@ def load_adapter(self, adapter_qualified_name: str): if err is not None: FancyLogger.get_logger().error(err) - raise Exception(f"error adding adapter: {err}") + raise Exception(f"error loading adapter {adapter_qualified_name}: {err}") + + self._loaded_adapters[adapter.qualified_name] = adapter def unload_adapter(self, adapter_qualified_name: str): # Unloads the adapter... - adapter = self._aloras.get(adapter_qualified_name, None) + # Check if the backend knows about this adapter. + adapter = self._added_adapters.get(adapter_qualified_name, None) if adapter is None: FancyLogger.get_logger().info( - f"could not unload adapter {adapter_qualified_name} for backend {self}: adapter was not previously added" + f"could not unload adapter {adapter_qualified_name} for backend {self}: adapter is not loaded" ) return - url = f"{self._base_url}/load_lora_adapter" + url = f"{self._base_url}/unload_lora_adapter" response = requests.post( url, json={"lora_name": adapter_qualified_name}, headers={"Content-Type": "application/json"}, ) - # TODO: JAL. Test the response codes if already unloaded. - # err: str | None = None - # match response.status_code: - # case 200: - # FancyLogger.get_logger().info( - # f"{url}: status {response.status_code} {response.text}" - # ) - # case 400: - # if "has already been loaded." in str(response.content): - # FancyLogger.get_logger().warning( - # f"{url}: status {response.status_code} {response.text}" - # ) - # else: - # err = f"{url}: status {response.status_code} {response.text}" - # case _: - # err = f"{url}: status {response.status_code} {response.text}" - - # if err is not None: - # FancyLogger.get_logger().error(err) - # raise Exception(f"error adding adapter: {err}") - - def add_alora(self, alora: "OpenAIAlora"): - """Loads an ALora for this backend. - - Args: - alora (str): identifier for the ALora adapter - """ - assert issubclass(alora.__class__, OpenAIAlora), ( - f"cannot add an ALora of type {alora.__class__} to model; must inherit from {OpenAIAlora.__class__}" - ) - assert alora._backend == self, "Cannot load an ALora into the wrong backend." - - if self.get_alora(alora.name) is not None: - FancyLogger.get_logger().warning( - f"Client code attempted to add {alora.name} but {alora.name} was already added to {self.__class__}. The backend is refusing to do this, because ALora loading is not idempotent." - ) - return None - - assert _server_type(self._base_url) == _ServerType.LOCALHOST, ( - "alora is supported only for locally running vllm instances" - ) - - # TODO: JAL. Make sure all hf model ids fit this way. - # base_model_name = self._hf_model_id.split("/")[1] - # TODO: JAL. change snapshot path to this...? - # snapshot_path = granite_common.intrinsics.util.obtain_lora( - # alora.name, base_model_name, alora=True - # ) - snapshot_path = f"/u/jakelorocco/eiger-user-folder/mellea-public/test/backends/test_openai_vllm/rag-intrinsics-lib/{alora.name}/alora/granite-3.3-8b-instruct" - - # https://docs.vllm.ai/en/stable/features/lora.html#using-api-endpoints - # curl -X POST http://localhost:8000/v1/load_lora_adapter \ - # -H "Content-Type: application/json" \ - # -d '{ - # "lora_name": "sql_adapter", - # "lora_path": "/path/to/sql-lora-adapter" - # }' - - url = f"{self._base_url}/load_lora_adapter" - response = requests.post( - url, - json={"lora_name": alora.name, "lora_path": snapshot_path}, - headers={"Content-Type": "application/json"}, - ) - - # TODO: Add a check here for the lora/alora already being loaded. - # TODO: See what happens if you try load the lora with the same name... - # TODO: If the alora isn't loaded, we should raise an error or at least not - # add it to the list of aloras. match response.status_code: case 200: FancyLogger.get_logger().info( f"{url}: status {response.status_code} {response.text}" ) - self._aloras[alora.name] = alora + case 404: + # This response code indicates that the adapter isn't currently loaded; + # which is the goal of this function. Log it but proceed as if successful. + FancyLogger.get_logger().info( + f"{url}: status {response.status_code} {response.text}" + ) case _: + # Unknown err. FancyLogger.get_logger().error( f"{url}: status {response.status_code} {response.text}" ) + raise Exception( + f"error unloading adapter {adapter_qualified_name}: {url}: status {response.status_code} {response.text}" + ) - self._aloras[alora.name] = alora - - return None - - def get_alora(self, alora_name: str) -> Alora | None: - """Returns the ALora by name, or None if that ALora isn't loaded.""" - return self._aloras.get(alora_name) - - def get_aloras(self) -> list[Alora]: - """Returns a list of all loaded ALora adapters.""" - return list(self._aloras.values()) + # Remove the alora from the list of loaded adapters. + del self._loaded_adapters[adapter.qualified_name] + + # TODO: JAL. Remove these functions. + # def add_alora(self, alora: "OpenAIAlora"): + # """Loads an ALora for this backend. + + # Args: + # alora (str): identifier for the ALora adapter + # """ + # assert issubclass(alora.__class__, OpenAIAlora), ( + # f"cannot add an ALora of type {alora.__class__} to model; must inherit from {OpenAIAlora.__class__}" + # ) + # assert alora._backend == self, "Cannot load an ALora into the wrong backend." + + # if self.get_alora(alora.name) is not None: + # FancyLogger.get_logger().warning( + # f"Client code attempted to add {alora.name} but {alora.name} was already added to {self.__class__}. The backend is refusing to do this, because ALora loading is not idempotent." + # ) + # return None + + # assert _server_type(self._base_url) == _ServerType.LOCALHOST, ( + # "alora is supported only for locally running vllm instances" + # ) + + # # TODO: JAL. Make sure all hf model ids fit this way. + # # base_model_name = self._hf_model_id.split("/")[1] + # # TODO: JAL. change snapshot path to this...? + # # snapshot_path = granite_common.intrinsics.util.obtain_lora( + # # alora.name, base_model_name, alora=True + # # ) + # snapshot_path = f"/u/jakelorocco/eiger-user-folder/mellea-public/test/backends/test_openai_vllm/rag-intrinsics-lib/{alora.name}/alora/granite-3.3-8b-instruct" + + # # https://docs.vllm.ai/en/stable/features/lora.html#using-api-endpoints + # # curl -X POST http://localhost:8000/v1/load_lora_adapter \ + # # -H "Content-Type: application/json" \ + # # -d '{ + # # "lora_name": "sql_adapter", + # # "lora_path": "/path/to/sql-lora-adapter" + # # }' + + # url = f"{self._base_url}/load_lora_adapter" + # response = requests.post( + # url, + # json={"lora_name": alora.name, "lora_path": snapshot_path}, + # headers={"Content-Type": "application/json"}, + # ) + + # # TODO: Add a check here for the lora/alora already being loaded. + # # TODO: See what happens if you try load the lora with the same name... + # # TODO: If the alora isn't loaded, we should raise an error or at least not + # # add it to the list of aloras. + # match response.status_code: + # case 200: + # FancyLogger.get_logger().info( + # f"{url}: status {response.status_code} {response.text}" + # ) + # self._aloras[alora.name] = alora + # case _: + # FancyLogger.get_logger().error( + # f"{url}: status {response.status_code} {response.text}" + # ) + + # self._aloras[alora.name] = alora + + # return None + + # def get_alora(self, alora_name: str) -> Alora | None: + # """Returns the ALora by name, or None if that ALora isn't loaded.""" + # return self._aloras.get(alora_name) + + # def get_aloras(self) -> list[Alora]: + # """Returns a list of all loaded ALora adapters.""" + # return list(self._aloras.values()) def apply_chat_template(self, chat: list[dict[str, str]]): """Apply the chat template for the model, if such a model is available (e.g., when it can deduce the huggingface model id).""" diff --git a/mellea/stdlib/base.py b/mellea/stdlib/base.py index fe565e691..e6f185904 100644 --- a/mellea/stdlib/base.py +++ b/mellea/stdlib/base.py @@ -114,7 +114,7 @@ def __repr__(self): return f"ImageBlock({self._value}, {self._meta.__repr__()})" -# TODO: JAL. Make this a CBlock? +# TODO: JAL. Make this a CBlock? it should be a CBlock with two fields... # TODO: JAL. Add support for passing in docs as model options? class Document: def __init__(self, text: str, title: str | None = None): diff --git a/mellea/stdlib/intrinsics/intrinsic.py b/mellea/stdlib/intrinsics/intrinsic.py index 8e88f88ab..c12befd48 100644 --- a/mellea/stdlib/intrinsics/intrinsic.py +++ b/mellea/stdlib/intrinsics/intrinsic.py @@ -45,7 +45,7 @@ def parts(self) -> list[Component | CBlock]: """The set of all the constituent parts of the `Intrinsic`. Will need to be implemented by subclasses since not all intrinsics are output - as text / messages. TODO: JAL. + as text / messages. """ raise NotImplementedError("parts isn't implemented by default") From 028600330a89904f7e60b5150dfd1cce1cbdad81 Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Sun, 2 Nov 2025 23:00:16 -0500 Subject: [PATCH 09/38] feat: update vllm serve script for granite common intrinsic downloads --- test/backends/test_openai_vllm/environment.yml | 1 + test/backends/test_openai_vllm/serve.sh | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/test/backends/test_openai_vllm/environment.yml b/test/backends/test_openai_vllm/environment.yml index 2d0b9e8e4..5ca4116b3 100644 --- a/test/backends/test_openai_vllm/environment.yml +++ b/test/backends/test_openai_vllm/environment.yml @@ -9,3 +9,4 @@ dependencies: variables: VLLM_USE_PRECOMPILED: 1 # need this flag for alora fork, installation fails otherwise VLLM_ALLOW_RUNTIME_LORA_UPDATING: True # allow loading (a)lora through POST http://localhost:8000/v1/load_lora_adapter + VLLM_DOWNLOAD_RAG_INTRINSICS: False # if True, download the rag-intrinsics-lib (https://huggingface.co/ibm-granite/rag-intrinsics-lib/tree/main); only required for remote vllm servers diff --git a/test/backends/test_openai_vllm/serve.sh b/test/backends/test_openai_vllm/serve.sh index 7746eed19..9b061d0e9 100755 --- a/test/backends/test_openai_vllm/serve.sh +++ b/test/backends/test_openai_vllm/serve.sh @@ -26,8 +26,18 @@ # see environment.yml. export VLLM_ALLOW_RUNTIME_LORA_UPDATING=True +# Mellea makes assumptions about the location of the adapter files on the server. By default, it assumes +# referenced adapters are at `./rag-intrinsics-lib/$adapter_name/$adapter_type/$base_model_name`. You can +# change this behavior by defining custom adapter classes that override the path. +# You will also need to set the OpenAIBackend's server_type to REMOTE_VLLM. +if [[ "$VLLM_ALLOW_RUNTIME_LORA_UPDATING" == "True" ]] && [[ "$VLLM_DOWNLOAD_RAG_INTRINSICS" == "True" ]] +then + echo "downloading rag-intrinsics-lib from huggingface" + hf download ibm-granite/rag-intrinsics-lib --local-dir ./rag-intrinsics-lib +fi + echo "launching a vllm server. Logs are found in $(readlink -ef $(dirname $0))/vllm.log" -vllm serve ibm-granite/granite-3.2-8b-instruct \ +vllm serve ibm-granite/granite-3.3-8b-instruct \ --enable-activated-lora \ --enable-lora \ --dtype bfloat16 \ From 12292065009bf82c9e0e40277a1588687b261eff Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Mon, 3 Nov 2025 09:07:41 -0500 Subject: [PATCH 10/38] feat: add intrinsics; remove bespoke aloras --- mellea/backends/_utils.py | 25 -- mellea/backends/adapters/adapter.py | 39 +- mellea/backends/aloras/__init__.py | 72 --- .../backends/aloras/huggingface/__init__.py | 1 - .../aloras/huggingface/granite_aloras.py | 309 ------------- mellea/backends/aloras/openai/__init__.py | 1 - .../backends/aloras/openai/granite_aloras.py | 143 ------ mellea/backends/huggingface.py | 409 +++++++++--------- mellea/backends/openai.py | 321 ++++++-------- mellea/stdlib/base.py | 34 +- mellea/stdlib/chat.py | 13 +- mellea/stdlib/intrinsics/intrinsic.py | 11 +- mellea/stdlib/requirement.py | 49 ++- test/backends/test_huggingface.py | 31 +- test/backends/test_huggingface_tools.py | 1 - .../test_openai_vllm/test_openai_vllm.py | 30 +- test/stdlib_basics/test_base.py | 1 - test/stdlib_basics/test_chat.py | 25 ++ 18 files changed, 491 insertions(+), 1024 deletions(-) delete mode 100644 mellea/backends/aloras/__init__.py delete mode 100644 mellea/backends/aloras/huggingface/__init__.py delete mode 100644 mellea/backends/aloras/huggingface/granite_aloras.py delete mode 100644 mellea/backends/aloras/openai/__init__.py delete mode 100644 mellea/backends/aloras/openai/granite_aloras.py create mode 100644 test/stdlib_basics/test_chat.py diff --git a/mellea/backends/_utils.py b/mellea/backends/_utils.py index c6e90ba8e..08720bc00 100644 --- a/mellea/backends/_utils.py +++ b/mellea/backends/_utils.py @@ -4,7 +4,6 @@ from collections.abc import Callable from typing import Any, Literal -from mellea.backends.aloras import Alora from mellea.backends.formatter import Formatter from mellea.backends.tools import parse_tools from mellea.helpers.fancy_logger import FancyLogger @@ -57,30 +56,6 @@ def to_chat( return ctx_as_conversation -def use_alora( - action: Component | CBlock, - alora: Alora | None, - default_to_constraint_checking_alora: bool, -) -> bool: - """Returns True when the condition for using alora is met. - - See `docs/dev/requirement_aLoRA_rerouting.md` for an explanation of the following code block. - """ - if issubclass(type(action), Requirement): - # The general rule is that we reroute to the alora if it exists. - reroute_to_alora = alora is not None - # However, there are some exceptions: - if not default_to_constraint_checking_alora: - reroute_to_alora = False - if issubclass(type(action), LLMaJRequirement): - reroute_to_alora = False - if issubclass(type(action), ALoraRequirement): - reroute_to_alora = True - return reroute_to_alora - else: - return False - - def to_tool_calls( tools: dict[str, Callable], decoded_result: str ) -> dict[str, ModelToolCall] | None: diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 2621dd8f2..7b7236614 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -1,7 +1,9 @@ +"""Module for adapters to backends.""" + import abc import pathlib from enum import Enum -from typing import TypeVar +from typing import Any, TypeVar import granite_common from litellm import cast @@ -11,11 +13,15 @@ class AdapterType(Enum): + """Possible types of adapters for a backend.""" + LORA = "lora" ALORA = "alora" class Adapter(abc.ABC): + """An adapter that can be added to a single backend.""" + def __init__(self, name: str, adapter_type: AdapterType): """An adapter that can be added to a backend. @@ -38,6 +44,8 @@ def __init__(self, name: str, adapter_type: AdapterType): class OpenAIAdapter(Adapter): + """Adapter for OpenAIBackends.""" + @abc.abstractmethod def get_open_ai_path( self, @@ -56,6 +64,8 @@ def get_open_ai_path( class LocalHFAdapter(Adapter): + """Adapter for LocalHFBackends.""" + @abc.abstractmethod def get_local_hf_path(self, base_model_name: str) -> str: """Returns the path needed to load the adapter. @@ -67,6 +77,8 @@ def get_local_hf_path(self, base_model_name: str) -> str: class GraniteCommonAdapter(OpenAIAdapter, LocalHFAdapter): + """Adapter for intrinsics that utilize the GraniteCommon library.""" + def __init__( self, name: str, @@ -122,6 +134,13 @@ def get_open_ai_path( server_type: _ServerType = _ServerType.LOCALHOST, remote_path: str | None = None, ) -> str: + """Returns the path needed to load the adapter. + + Args: + base_model_name: the base model; typically the last part of the huggingface model id like "granite-3.3-8b-instruct" + server_type: the server type (ie LOCALHOST / OPENAI); usually the backend has information on this + remote_path: optional; used only if the server_type is REMOTE_VLLM; base path at which to find the adapter + """ if server_type == _ServerType.LOCALHOST: path = self.download_and_get_path(base_model_name) elif server_type == _ServerType.REMOTE_VLLM: @@ -136,6 +155,11 @@ def get_open_ai_path( return path def get_local_hf_path(self, base_model_name: str) -> str: + """Returns the path needed to load the adapter. + + Args: + base_model_name: the base model; typically the last part of the huggingface model id like "granite-3.3-8b-instruct" + """ return self.download_and_get_path(base_model_name) def download_and_get_path(self, base_model_name: str) -> str: @@ -185,3 +209,16 @@ def get_adapter_for_intrinsic( break return adapter + + +class AdapterMixin(abc.ABC): + """Mixin class for backends capable of utilizing adapters.""" + + def add_adapter(self, *args, **kwargs): + """Adds the given adapter to the backend. Must not have been added to a different backend.""" + + def load_adapter(self, adapter_qualified_name: str): + """Loads the given adapter for the backend. Must have previously been added.""" + + def unload_adapter(self, adapter_qualified_name: str): + """Unloads the given adapter from the backend.""" diff --git a/mellea/backends/aloras/__init__.py b/mellea/backends/aloras/__init__.py deleted file mode 100644 index 9f8cc3494..000000000 --- a/mellea/backends/aloras/__init__.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Abstract interfaces for Backends that implement Activated LoRAs.""" - -import abc - -from mellea.backends.adapters.adapter import Adapter -from mellea.stdlib.base import CBlock, ModelOutputThunk - - -# TODO: JAL. Remove this. -class Alora(abc.ABC): - """Activated LoRAs (Aloras)](https://arxiv.org/pdf/2504.12397) are are [low-rank adapters](https://arxiv.org/abs/2106.09685) that can reuse KV cache from their underlying model. - - This class should not be directly subclassed by a specific ALora. Each backend that supports ALora should provide a backend-specific abstract class that subclasses `ALora`. Individual ALoras should then be defined by subclassing the model-specific backend. - - ALoras are always attached to an underlying model and use the following calling convention: - 1. The underlying model is prompted (without the Alora active). We call this the `input`. - 2. The underlying model generates some tokens from the `input` context (again, without the ALora active). We call this the `response`. - 3. Then the adapter is activated and generates some tokens. We call then the `alora_response`. - - Args: - name (str): An arbitrary name/label in the model serving engine (e.g. vllm, or local huggingface) to assign to an ALora. This is irrelevant from the alora's (huggingface) model id. - """ - - def __init__(self, name: str): - """Each aLoRA is identified by a name.""" - self.name: str = name - - @abc.abstractmethod - def generate_using_strings(self, *args, **kwargs) -> ModelOutputThunk: - """Generates from the ALora using raw strings as the interface for inputs. In most cases, must be run from a running event loop. - - This has a generic signature because each aLoRA has different parameters depending on its functionality and how it gets called. - """ - - def generate_using_stdlib(self, *args, **kwargs) -> CBlock: - """Generates from the Alora using Span-based backends.""" - # This is NOT marked as an `abc.abstractmethod` for now because we are not releasing span-based backends. When we release a span-based backend, we should mark this method as `abc.abstractmethod`""" - raise NotImplementedError( - "There are not currently ant ALoras trained to use spans." - ) - - -class AloraBackendMixin(abc.ABC): - """Mixin class for backends capable of aLoRA functionality.""" - - @abc.abstractmethod - def add_alora(self, *args, **kwargs): - """Loads an ALora.""" - ... - - @abc.abstractmethod - def get_alora(self, alora_name: str) -> Alora | None: - """Returns the ALora by name, or None of that ALora isn't loaded.""" - ... - - @abc.abstractmethod - def get_aloras(self) -> list[Alora]: - """Returns a list of all loaded aLoRA adapters.""" - ... - - -class AdapterMixin(abc.ABC): - """Mixin class for backends capable of utilizing adapters.""" - - def add_adapter(self, *args, **kwargs): - """Adds the given adapter to the backend. Must not have been added to a different backend.""" - - def load_adapter(self, adapter_qualified_name: str): - """Loads the given adapter for the backend. Must have previously been added.""" - - def unload_adapter(self, adapter_qualified_name: str): - """Unloads the given adapter from the backend.""" diff --git a/mellea/backends/aloras/huggingface/__init__.py b/mellea/backends/aloras/huggingface/__init__.py deleted file mode 100644 index 6746ef506..000000000 --- a/mellea/backends/aloras/huggingface/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""ALora implementations for `mellea.backends.huggingface` backends.""" diff --git a/mellea/backends/aloras/huggingface/granite_aloras.py b/mellea/backends/aloras/huggingface/granite_aloras.py deleted file mode 100644 index 22809ed97..000000000 --- a/mellea/backends/aloras/huggingface/granite_aloras.py +++ /dev/null @@ -1,309 +0,0 @@ -"""Huggingface implementations for IBM's "starter pack" of Activated LoRAs.""" - -import asyncio -import functools -from copy import deepcopy - -import granite_common -import torch -from transformers.generation.utils import GenerateDecoderOnlyOutput - -from mellea.backends.huggingface import HFAlora, HFAloraCacheInfo, LocalHFBackend -from mellea.backends.types import ModelOption -from mellea.helpers.async_helpers import send_to_queue -from mellea.helpers.fancy_logger import FancyLogger -from mellea.stdlib.base import GenerateType, ModelOutputThunk - - -class HFConstraintAlora(HFAlora): - """The Requirement Checking ALora for Granite checks if the specified requirement was satisfied by the most recent model generation. Only one requirement is checked at a time. - - Currently supports [Granite 3.2 8B](https://huggingface.co/ibm-granite/granite-3.2-8b-alora-requirement-check) and [Granite 3.3 8B](https://huggingface.co/ibm-granite/granite-3.3-8b-alora-requirement-check) by default. - """ - - def __init__( - self, - name: str, - path_or_model_id: str, - generation_prompt: str, - backend: LocalHFBackend, - *, - constraint_prompt: str | None = None, - include_constraint_in_alora_offset: bool = False, - ): - """Initialize after checking that the backend is correct. - - Args: - name: name of the alora. - path_or_model_id: huggingface path or model id. - generation_prompt: the prompt required to activate the aLoRa. - backend: a LocalHFBackend that this alora is attached to. - constraint_prompt: a template that the constraint can be interpolated into; can only have a single `{}` slot. - include_constraint_in_alora_offset: whether to include the constraint prompt in the alora offset. - """ - super().__init__(name, path_or_model_id, generation_prompt, backend) - - # Maintain default behavior. - if constraint_prompt is None: - constraint_prompt = "\nRequirement: {}<|end_of_text|>\n" - - self._constraint_prompt = constraint_prompt - self._include_constraint_in_alora_offset = include_constraint_in_alora_offset - - # We do a lot of logging for ALoras because this is an experimental feature. Maybe we should tag these log messages? - self._logger = FancyLogger.get_logger() - - def generate_using_strings( - self, - input: str, - response: str, - constraint: str, - force_yn: bool = True, - stream: bool = False, - ) -> ModelOutputThunk: - """Generates a constraint response from the ALora. Must be run in a running event loop.""" - # Go ahead and do runtime type-checking because passing CBlocks into this function is a common error. - assert type(input) is str - assert type(response) is str - assert type(constraint) is str - self._backend._model.set_adapter(self.name) - cache_hit = self._backend.cache_get(response) - - if stream: - self._logger.warning( - "`HFConstraintAlora` cannot stream output; defaulting to non-streaming approach." - ) - - generate_kwargs = {} - if cache_hit: - self._logger.debug( - f"using cache for alora {self.__class__} and response '{response}'" - ) - generate_kwargs["past_key_values"] = deepcopy(cache_hit.kv_cache) - input_combined = self._generate_using_cache(cache_hit, constraint, force_yn) - - else: - self._logger.debug( - f"not using cache for alora {self.__class__} and response '{response}'" - ) - input_combined = self._generate_not_using_cache( - input, response, constraint, force_yn - ) - - if not self._include_constraint_in_alora_offset: - alora_offsets = [self._generation_prompt_tokens["input_ids"].shape[1] - 1] - else: - # Get the constraint tokens separately so that we can calculate the alora offsets. - constraint_tokens = self._backend._tokenizer( - self._constraint_prompt.format(constraint), return_tensors="pt" - ) - - alora_offsets = [ - constraint_tokens["input_ids"].shape[1] - + self._generation_prompt_tokens["input_ids"].shape[1] - - 2 - ] - - chat_response = asyncio.to_thread( - self._backend._model.generate, # type: ignore - input_combined["input_ids"], - attention_mask=input_combined["attention_mask"], - max_new_tokens=1, - return_dict_in_generate=True, - alora_offsets=alora_offsets, - output_scores=True, - **generate_kwargs, - ) - - output = ModelOutputThunk(None) - output._meta["alora_name"] = self.name - - output._process = functools.partial( - processing, - backend=self._backend, - force_yn=force_yn, - gen_prompt=self._generation_prompt, - ) - output._post_process = functools.partial(post_processing, backend=self._backend) - - try: - # To support lazy computation, will need to remove this create_task and store just the unexecuted coroutine. - # We can also support synchronous calls by adding a flag and changing this ._generate function. - - # This function should always be called from a running event loop so we don't have to worry about - # scheduling the task to a specific event loop here. - output._generate = asyncio.create_task( - send_to_queue(chat_response, output._async_queue) # type: ignore - ) - output._generate_type = GenerateType.ASYNC - except RuntimeError as e: - # Most likely cause is running this function without an event loop present. - raise e - - return output - - def _generate_using_cache( - self, cache_hit: HFAloraCacheInfo, constraint: str, force_yn: bool - ) -> dict: - """Returns the input object used for generation.""" - # Must tokenize the constraint here since the requirement isn't known at initialization. - constraint_tokens = self._backend._tokenizer( - self._constraint_prompt.format(constraint), return_tensors="pt" - ) - - input_combined = { - "input_ids": torch.cat( - [ - cache_hit.merged_token_ids.unsqueeze(0), - constraint_tokens["input_ids"], - self._generation_prompt_tokens["input_ids"], - ], - dim=1, - ), - "attention_mask": torch.cat( - [ - cache_hit.merged_attention.unsqueeze(0), - constraint_tokens["attention_mask"], - self._generation_prompt_tokens["attention_mask"], - ], - dim=1, - ), - } - - self._logger.debug( - f"Prompt for cached aLoRA({self.name}):\n {self._backend._tokenizer.decode(input_combined['input_ids'][0])}" - ) - - return input_combined - - def _generate_not_using_cache( - self, input: str, response: str, constraint: str, force_yn: bool - ) -> dict: - """Returns the input object used for generation.""" - # Params aren't needed when just getting the backend args. - backend_model_opts = self._backend._simplify_and_merge(None) - sys_prompt = backend_model_opts.get(ModelOption.SYSTEM_PROMPT, None) - - chat = [ - *([{"role": "system", "content": sys_prompt}] if sys_prompt else []), - {"role": "user", "content": input}, - {"role": "assistant", "content": response}, - ] - - templatized = self._backend._tokenizer.apply_chat_template(chat, tokenize=False) - assert type(templatized) is str - - # Must tokenize the constraint here since the requirement isn't known at initialization. - templatized = templatized + self._constraint_prompt.format(constraint) - - tokenized = self._backend._tokenizer(templatized, return_tensors="pt") - input_combined = { - "input_ids": torch.cat( - [tokenized["input_ids"], self._generation_prompt_tokens["input_ids"]], - dim=1, - ), - "attention_mask": torch.cat( - [ - tokenized["attention_mask"], - self._generation_prompt_tokens["attention_mask"], - ], - dim=1, - ), - } - - self._logger.debug( - f"Prompt for non-cached aLoRA({self.name}):\n{self._backend._tokenizer.decode(input_combined['input_ids'][0])}" - ) - - return input_combined - - -async def processing( - mot: ModelOutputThunk, - chunk: GenerateDecoderOnlyOutput, - backend: LocalHFBackend, - force_yn: bool, - gen_prompt: str, -): - """Called to process the incoming chunks.""" - if mot._underlying_value is None: - mot._underlying_value = "" - - # Don't support async for HFConstraintAlora. Means we can process the output here. - assert isinstance(chunk, GenerateDecoderOnlyOutput) - - if force_yn: - last_logits = chunk.scores[-1].squeeze(0) # type: ignore - token_Y = backend._tokenizer("Y", add_special_tokens=False)["input_ids"][0] # type: ignore - token_N = backend._tokenizer("N", add_special_tokens=False)["input_ids"][0] # type: ignore - logit_Y = last_logits[token_Y].item() - logit_N = last_logits[token_N].item() - mot._underlying_value = "Y" if logit_Y > logit_N else "N" - else: - output_text = backend._tokenizer.decode(chunk.sequences[0]) - constraint_satisfied = output_text.split(gen_prompt)[-1] - mot._underlying_value = constraint_satisfied[ - 0 - ] # Grab the first char of the str. - - -async def post_processing(mot: ModelOutputThunk, backend: LocalHFBackend): - """Called after all data has been received.""" - backend.formatter.parse(mot._action, mot) # type: ignore - - -def add_granite_aloras(backend: LocalHFBackend): - """Adds the IBM Granite "starter pack" ALoras to a backend.""" - if backend._hf_model_id == "ibm-granite/granite-3.2-8b-instruct": - backend.add_alora( - HFConstraintAlora( - name="constraint", - path_or_model_id="ibm-granite/granite-3.2-8b-alora-requirement-check", - generation_prompt="<|start_of_role|>check_requirement<|end_of_role|>", - backend=backend, - constraint_prompt="\nRequirement: {}<|end_of_text|>\n", - include_constraint_in_alora_offset=False, - ) - ) - elif backend._hf_model_id == "ibm-granite/granite-3.3-8b-instruct": - backend.add_alora( - HFConstraintAlora( - name="constraint", - path_or_model_id="ibm-granite/granite-3.3-8b-alora-requirement-check", - generation_prompt="<|start_of_role|>check_requirement<|end_of_role|>", - backend=backend, - constraint_prompt="\n<|start_of_role|>requirement<|end_of_role|>{}<|end_of_text|>\n", - include_constraint_in_alora_offset=True, - ) - ) - else: - raise ValueError( - f"cannot add_granite_aloras to unknown huggingface model_id / backend: {backend._hf_model_id}" - ) - - -class HFIntrinsicAlora(HFAlora): - def __init__( - self, name: str, backend: LocalHFBackend, base_model_name: str | None = None - ): - # TODO: JAL. May need to change HFAlora to not need a generation prompt by default... - - if base_model_name is None: - # TODO: JAL. Make sure all hf model ids fit this way. - base_model_name = backend._hf_model_id.split("/")[1] - - # TODO: JAL. figure out if we want to download files here or when adding the alora... - # Need to do all the path determination at init? - path = str( - granite_common.intrinsics.util.obtain_lora( - name, base_model_name, alora=True - ) - ) - - super().__init__(name, path, "", backend) - - # We do a lot of logging for ALoras because this is an experimental feature. Maybe we should tag these log messages? - self._logger = FancyLogger.get_logger() - - def generate_using_strings(self, *args, **kwargs) -> ModelOutputThunk: - raise NotImplementedError() diff --git a/mellea/backends/aloras/openai/__init__.py b/mellea/backends/aloras/openai/__init__.py deleted file mode 100644 index f07c7f75b..000000000 --- a/mellea/backends/aloras/openai/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""ALora implementations for `mellea.backends.openai` backends.""" diff --git a/mellea/backends/aloras/openai/granite_aloras.py b/mellea/backends/aloras/openai/granite_aloras.py deleted file mode 100644 index a53e0f1ca..000000000 --- a/mellea/backends/aloras/openai/granite_aloras.py +++ /dev/null @@ -1,143 +0,0 @@ -"""OpenAI implementations for IBM's "starter pack" of Activated LoRAs.""" - -import asyncio -import functools -from collections.abc import Coroutine -from typing import Any - -import openai -from openai.types.completion import Completion - -from mellea.backends.aloras import Alora -from mellea.backends.openai import OpenAIAlora, OpenAIBackend -from mellea.backends.types import ModelOption -from mellea.helpers.async_helpers import send_to_queue -from mellea.helpers.fancy_logger import FancyLogger -from mellea.stdlib.base import GenerateType, ModelOutputThunk - - -# TODO: JAL. this should be made into a generic adapter interface...? -class OpenAIIntrinsicAlora(OpenAIAlora): - """The [Requirement Checking ALora for Granite 3.2 8B](https://huggingface.co/ibm-granite/granite-3.2-8b-alora-requirement-check) checks if the specified requirement was satisfied by the most recent model generation. Only one requirement is checked at a time.""" - - def __init__( - self, name: str, path: str, generation_prompt: str, backend: OpenAIBackend - ): - """Initialize after checking that the backend is correct.""" - super().__init__(name, path, generation_prompt, backend) - # We do a lot of logging for ALoras because this is an experimental feature. Maybe we should tag these log messages? - self._logger = FancyLogger.get_logger() - # TODO: JAL. Move the adapter loading logic to the init?... or see where we want to make sure they are downloaded... - - def generate_using_strings( - self, - input: str, - response: str, - constraint: str, - force_yn: bool = True, - stream: bool = False, - ) -> ModelOutputThunk: - """Generates a constraint response from the ALora. Must be run in a running event loop.""" - # Go ahead and do runtime type-checking because passing CBlocks into this function is a common error. - assert type(input) is str - assert type(response) is str - assert type(constraint) is str - - # Params aren't needed when just getting the backend args. - backend_model_opts = self._backend._simplify_and_merge(None, False) - sys_prompt = backend_model_opts.get(ModelOption.SYSTEM_PROMPT, None) - - chat = [ - *([{"role": "system", "content": sys_prompt}] if sys_prompt else []), - {"role": "user", "content": input}, - {"role": "assistant", "content": response}, - ] - - prompt = self._backend.apply_chat_template(chat) - prompt += f"\nRequirement: {constraint}<|end_of_text|>\n" # type: ignore - prompt += self._generation_prompt - - self._logger.debug(f"Prompt for non-cached aLoRA({self.name}):\n{prompt}") - - force_yn_args = {} - if force_yn: - assert hasattr(self._backend, "_tokenizer") - token_Y = self._backend._tokenizer("Y", add_special_tokens=False)[ - "input_ids" - ][0] # type: ignore - token_N = self._backend._tokenizer("N", add_special_tokens=False)[ - "input_ids" - ][0] # type: ignore - - force_yn_args["logit_bias"] = {str(token_Y): 100, str(token_N): 100} - - chat_response: Coroutine[ - Any, Any, openai.AsyncStream[Completion] | Completion - ] = self._backend._async_client.completions.create( - model=self.name, - prompt=prompt, - max_tokens=1, - n=1, - stream=stream, - **force_yn_args, - ) # type: ignore - - output = ModelOutputThunk(None) - output._meta["alora_name"] = self.name - - output._process = processing - output._post_process = functools.partial(post_processing, self._backend) - - try: - # To support lazy computation, will need to remove this create_task and store just the unexecuted coroutine. - # We can also support synchronous calls by adding a flag and changing this ._generate function. - - # This function should always be called from a running event loop so we don't have to worry about - # scheduling the task to a specific event loop here. - output._generate = asyncio.create_task( - send_to_queue(chat_response, output._async_queue) - ) - output._generate_type = GenerateType.ASYNC - except RuntimeError as e: - # Most likely cause is running this function without an event loop present - raise e - - return output - - -async def processing(mot: ModelOutputThunk, chunk: Completion): - """Called to process the incoming chunks.""" - if mot._underlying_value is None: - mot._underlying_value = "" - mot._underlying_value += chunk.choices[0].text - - -async def post_processing(backend: OpenAIBackend, mot: ModelOutputThunk): - """Called after all data has been received.""" - backend.formatter.parse(mot._action, mot) # type: ignore - - -def add_granite_aloras(backend: OpenAIBackend): - """Adds the IBM Granite "starter pack" ALoras to a backend.""" - if backend._hf_model_id == "ibm-granite/granite-3.2-8b-instruct": - backend.add_alora( - OpenAIIntrinsicAlora( - name="constraint", - path="ibm-granite/granite-3.2-8b-alora-requirement-check", - generation_prompt="<|start_of_role|>check_requirement<|end_of_role|>", - backend=backend, - ) - ) - elif backend._hf_model_id == "ibm-granite/granite-3.3-8b-instruct": - backend.add_alora( - OpenAIIntrinsicAlora( - name="constraint", - path="ibm-granite/granite-3.3-8b-alora-requirement-check", - generation_prompt="<|start_of_role|>check_requirement<|end_of_role|>", - backend=backend, - ) - ) - else: - raise ValueError( - f"cannot add_granite_aloras to unknown huggingface model_id / backend: {backend._hf_model_id}" - ) diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index 5e3f073e5..660d381df 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -33,9 +33,14 @@ from transformers.generation.utils import GenerateDecoderOnlyOutput from mellea.backends import BaseModelSubclass -from mellea.backends._utils import to_chat, to_tool_calls, use_alora -from mellea.backends.adapters.adapter import GraniteCommonAdapter, LocalHFAdapter -from mellea.backends.aloras import Alora, AloraBackendMixin +from mellea.backends._utils import to_chat, to_tool_calls +from mellea.backends.adapters.adapter import ( + AdapterMixin, + AdapterType, + GraniteCommonAdapter, + LocalHFAdapter, + get_adapter_for_intrinsic, +) from mellea.backends.cache import Cache, SimpleLRUCache from mellea.backends.formatter import Formatter, FormatterBackend, TemplateFormatter from mellea.backends.model_ids import ModelIdentifier @@ -83,7 +88,7 @@ class HFAloraCacheInfo: q_end: int = -1 -class LocalHFBackend(FormatterBackend, AloraBackendMixin): +class LocalHFBackend(FormatterBackend, AdapterMixin): """The LocalHFBackend uses Huggingface's transformers library for inference, and uses a Formatter to convert `Component`s into prompts. This backend also supports Activated LoRAs (ALoras)](https://arxiv.org/pdf/2504.12397). This backend is designed for running an HF model for small-scale inference locally on your machine. @@ -164,7 +169,9 @@ def __init__( self._use_caches = use_caches self._cache = cache if cache is not None else SimpleLRUCache(3) - self._aloras: dict[str, HFAlora] = {} + # Adapters can be made know to the backend (added) and loaded. + self._added_adapters: dict[str, LocalHFAdapter] = {} + self._loaded_adapters: dict[str, LocalHFAdapter] = {} def generate_from_context( self, @@ -179,86 +186,55 @@ def generate_from_context( # Upsert model options. model_opts = self._simplify_and_merge(model_options) - if use_alora( - action, - self.get_alora("constraint"), - self.default_to_constraint_checking_alora, - ): - mot = self._generate_from_context_alora( - action, ctx, _format=format, model_options=model_opts - ) - return mot, ctx.add(mot) - elif isinstance(action, Intrinsic): - mot = self._generate_from_intrinsic( - action, ctx, _format=format, model_options=model_opts - ) - return mot, ctx.add(mot) - else: - mot = self._generate_from_context_standard( - action, - ctx, - _format=format, - model_options=model_opts, - tool_calls=tool_calls, + # Requirements can be automatically rerouted to a requirement adapter. + if isinstance(action, Requirement): + # See docs/dev/requirement_aLoRA_rerouting.md + reroute_to_alora = self.default_to_constraint_checking_alora + adapter_name = "requirement_check" + + if isinstance(action, ALoraRequirement): + reroute_to_alora = True + adapter_name = action.intrinsic_name + alora_action = action + else: + assert action.description is not None, ( + "must have a description when generating from a requirement" + ) + alora_action = ALoraRequirement(action.description, adapter_name) + + # Check if a requirement_check (or AloraRequirement specified) adapter exists. + alora_req_adapter = get_adapter_for_intrinsic( + adapter_name, [AdapterType.ALORA], self._added_adapters ) - return mot, ctx.add(action).add(mot) + if alora_req_adapter is None: + # Log a warning if using an AloraRequirement but no adapter fit. + if reroute_to_alora: + FancyLogger.get_logger().warning( + f"attempted to use an AloraRequirement but backend {self} doesn't have the specified adapter added {adapter_name}; defaulting to regular generation" + ) - def _generate_from_context_alora( - self, - action: Component | CBlock, - ctx: Context, - *, - _format: type[BaseModelSubclass] | None = None, - model_options: dict[str, Any], - ) -> ModelOutputThunk: - match action: - case ALoraRequirement(): - alora_for_this_request = ( - self.get_alora("constraint") - if action.alora is None - else action.alora - ) - case _: - alora_for_this_request = self.get_alora("constraint") - assert alora_for_this_request is not None, ( - "This code block should not execute unless there is a 'constraint' alora loaded." - ) - # Construct the linearized context. This is very similar to normal generation. - linearized_ctx = ctx.view_for_generation() - assert linearized_ctx is not None and len(linearized_ctx) > 1 - msgs = self.formatter.to_chat_messages(linearized_ctx) - user_message, assistant_message = msgs[-2].content, msgs[-1].content - assert alora_for_this_request is not None - assert type(user_message) is str - assert type(assistant_message) is str - assert _format is None, "Structured outputs are not supported by ALoRAs." - - alora_output = alora_for_this_request.generate_using_strings( - input=user_message, - response=assistant_message, - constraint=action.description, # type: ignore - stream=model_options.get(ModelOption.STREAM, False), - ) + if issubclass(type(action), LLMaJRequirement): + reroute_to_alora = False - # The alora function doesn't set up all the fields. - alora_output._context = linearized_ctx - alora_output._action = action - alora_output._model_options = model_options + if reroute_to_alora: + # Keep the alora requirement handling separate for now. + mot = self._generate_from_intrinsic( + alora_action, ctx, model_options=model_opts + ) + return mot, ctx.add(alora_action).add(mot) - # TODO: Figure out what info we want to populate for aloras here. - alora_output._generate_log = GenerateLog() + elif isinstance(action, Intrinsic): + mot = self._generate_from_intrinsic(action, ctx, model_options=model_opts) + return mot, ctx.add(action).add(mot) - return alora_output + mot = self._generate_from_context_standard( + action, ctx, _format=format, model_options=model_opts, tool_calls=tool_calls + ) + return mot, ctx.add(action).add(mot) def _generate_from_intrinsic( - self, - action: Intrinsic, - ctx: Context, - *, - _format: type[BaseModelSubclass] | None = None, # TODO: JAL. Remove this param? - model_options: dict[str, Any], - tool_calls: bool = False, # TODO: JAL. Remove this param? - ) -> ModelOutputThunk: # TODO: JAL. Return context here as well? Else treat like Aloras and add the context higher up. + self, action: Intrinsic, ctx: Context, *, model_options: dict[str, Any] + ) -> ModelOutputThunk: if not ctx.is_chat_context: raise Exception("Does not yet support non-chat contexts.") @@ -270,64 +246,45 @@ def _generate_from_intrinsic( linearized_ctx ) - # TODO: JAL. Fix where this happens. - # add action - # ctx_as_message_list.extend(self.formatter.to_chat_messages([action])) - # ctx_as_conversation = [ - # {"role": m.role, "content": m.content} for m in ctx_as_message_list - # ] - - # Check that we ddin't accidentally end up with CBlocks. - # for msg in ctx_as_conversation: - # for v in msg.values(): - # if "CBlock" in v: - # FancyLogger.get_logger().error( - # f"Found the string `CBlock` in what should've been a stringified context: {ctx_as_conversation}" - # ) - conversation: list[dict] = [] system_prompt = model_options.get(ModelOption.SYSTEM_PROMPT, "") if system_prompt != "": conversation.append({"role": "system", "content": system_prompt}) - # TODO: JAL. Fix where this function lives. conversation.extend( [OpenAIBackend.message_to_openai_message(m) for m in ctx_as_message_list] ) - # handle custom system prompts. It's important that we do this before the _parse_and_**clean**_model_options step. - # system_prompt = model_options.get(ModelOption.SYSTEM_PROMPT, None) - # if system_prompt is not None: - # system_msg: dict[str, str] = { - # "role": "system", - # "content": system_prompt, - # } - # ctx_as_conversation.insert(0, system_msg) + docs = OpenAIBackend.messages_to_docs(ctx_as_message_list) seed = model_options.get(ModelOption.SEED, None) if seed is not None: set_seed(seed) - # TODO: JAL. Use the better matching function. Going by name alone right now. - # Also move this to be an early exit. - # TODO: JAL. Check for LORAs as well once that interface is done. - adapter = self.get_alora(action.intrinsic_name + "_" + "alora") + if model_options.get(ModelOption.STREAM, None) is not None: + # Intrinsics don't support streaming because of their post-processing step. + FancyLogger.get_logger().warning( + "intrinsics cannot use streaming; removing model option" + ) + del model_options[ModelOption.STREAM] + + adapter = get_adapter_for_intrinsic( + action.intrinsic_name, action.adapter_types, self._added_adapters + ) if adapter is None: - adapter = self.get_alora(action.intrinsic_name + "_" + "lora") - if adapter is None: - raise ValueError( - f"no alora / lora for processing intrinsic: {action.intrinsic_name}" - ) + raise ValueError( + f"backend ({self}) has no adapter for processing intrinsic: {action.intrinsic_name}" + ) - # TODO: JAL. Fix this once get_alora is changed. + # TODO: Code below this point is mostly specific to RagIntrinsics (and granite_common). + # It should be refactored into a specific adapter.transform() function. assert isinstance(adapter, GraniteCommonAdapter) - # TODO: Check that the base_model_name matches the backend's model? + intrinsic_config = adapter.config if intrinsic_config is None: # If the adapter wasn't initialized with a config, grab one here based off the backend's model. intrinsic_config_file = granite_common.intrinsics.util.obtain_io_yaml( - action.intrinsic_name, - self.model_id, # TODO: JAL. We need to a specific string here if model id isn't one...; probably the huggingface id + action.intrinsic_name, self._hf_model_id.split("/")[-1] ) intrinsic_config = granite_common.intrinsics.util.make_config_dict( config_file=intrinsic_config_file @@ -336,59 +293,103 @@ def _generate_from_intrinsic( dict, intrinsic_config ) # TODO: Can remove if util function gets exported properly. - # Check if there's a model name associated with this intrinsic (either user specified - # or specified in the config). - model_name = adapter.base_model_name - if model_name is None: - model_name = intrinsic_config["model"] - # TODO: JAL. model_name is only really used for routing the request. See what we do with - # vllm / openai with aloras. If the model name is the same and we just activate - # the adapters, that's fine; we can leave the model name alone... - # - with openai / vllm, you actually do use the lora adapter name... - # - with openai / vllm, you use the alora name... - # I think this also means no generation lock is required for OpenAI; only huggingface - - intrinsic_config["response_format"] = json.dumps( - intrinsic_config["response_format"] - ) - rewriter_config = deepcopy(intrinsic_config) - result_processor_config = deepcopy(intrinsic_config) - - # TODO: JAL. The name passed to the intrinsics rewriter should be the alora's or the intrinsics? theoretically - # the alora's should always be the intrinsic's given the matching func. rewriter = granite_common.IntrinsicsRewriter( - config_dict=rewriter_config, model_name=adapter.name + config_dict=intrinsic_config, model_name=adapter.name ) result_processor = granite_common.IntrinsicsResultProcessor( - config_dict=result_processor_config + config_dict=intrinsic_config ) # Convert our conversation into a proper chat completions dict. # [{role: user, content: Hello}, {...}] -> {messages: [{role:user,...}, ...], model:..., ...} - request_json: dict = {"messages": conversation} + request_json: dict = { + "messages": conversation, + "extra_body": {"documents": docs}, + } rewritten = rewriter.transform(request_json, **action.intrinsic_kwargs) - # TODO: JAL. Will need a lock here... + # TODO: Handle caching here. granite_common doesn't tell us what changed, + # so we will have to invalidate the cache on our side. This requires + # us having specific caching for each Component/Message. + + self.load_adapter(adapter.qualified_name) + + # TODO: This modifies the underlying model. We should set a non-exclusive lock here. + # It should allow generate requests with the same adapter to proceed. This logic also + # needs to be added to the other generate functions. self._model.set_adapter(adapter.qualified_name) + generate_input, other_input = ( granite_common.util.chat_completion_request_to_transformers_inputs( rewritten, self._tokenizer, self._model ) ) - # TODO: Will need to break this piece out to support caching... - chat_response = granite_common.util.generate_with_transformers( - self._tokenizer, self._model, generate_input, other_input + chat_response: Coroutine[Any, Any, granite_common.ChatCompletionResponse] = ( + asyncio.to_thread( + granite_common.util.generate_with_transformers, + self._tokenizer, + self._model, + generate_input, + other_input, + ) ) - processed_chat_completion = result_processor.transform(chat_response, rewritten) - # TODO: JAL. Put into ModelOutputThunk, parse, etc... - mot = ModelOutputThunk(processed_chat_completion.choices[0].message.content) - mot._generate_log = GenerateLog() + output = ModelOutputThunk(None) + output._context = ctx.view_for_generation() + output._action = action + output._model_options = model_options + + # Add another step to the processing function. + async def granite_common_processing( + mot: ModelOutputThunk, + chunk: granite_common.ChatCompletionResponse, + rewritten: granite_common.ChatCompletion, + result_processor: granite_common.IntrinsicsResultProcessor, + input_ids, + ): + res = result_processor.transform(chunk, rewritten) # type: ignore - # TODO: JAL. Move this deactivation elsewhere. - self._model.disable_adapters() - return mot + # TODO: If we want to support caches, we need to get the GenerateDecoderOnlyOutput. This means we + # probably need to break out the pieces from `generate_with_transformers`. + # processing expects a str or a GenerateDecoderOnlyOutput. Extract the str. + return await self.processing( + mot, res.choices[0].message.content, input_ids=input_ids + ) + + output._process = functools.partial( + granite_common_processing, + rewritten=rewritten, + result_processor=result_processor, + input_ids=generate_input["input_tokens"], + ) + + # TODO: Post-processing should release the lock for this generation. + output._post_process = functools.partial( + self.post_processing, + conversation=conversation, + input_ids=generate_input["input_tokens"], + _format=None, + tool_calls=False, + tools={}, + seed=seed, + ) + + try: + # To support lazy computation, will need to remove this create_task and store just the unexecuted coroutine. + # We can also support synchronous calls by adding a flag and changing this ._generate function. + + # This function should always be called from a running event loop so we don't have to worry about + # scheduling the task to a specific event loop here. + output._generate = asyncio.create_task( + send_to_queue(chat_response, output._async_queue) # type: ignore + ) + output._generate_type = GenerateType.ASYNC + except RuntimeError as e: + # Most likely cause is running this function without an event loop present. + raise e + + return output def _generate_from_context_standard( self, @@ -579,7 +580,7 @@ async def post_processing( assert mot.value is not None # Add an entry to the cache for ALora reuse. - if self._use_caches: + if self._use_caches and mot._meta.get("hf_output", None) is not None: output_complete = mot._meta["hf_output"].sequences[0] cache: DynamicCache = mot._meta["hf_output"].past_key_values # type: ignore @@ -793,78 +794,68 @@ def _filter_chat_template_only_options( } return {k: v for k, v in model_options.items() if k not in chat_template_only} - # region ALora loading, unloading, and utility functions. + # region Adapter loading, unloading, and utility functions. def add_adapter(self, adapter: LocalHFAdapter): - base_model_name = self._hf_model_id.split("/")[1] - path = adapter.get_local_hf_path(base_model_name) + """Adds the given adapter to the backend. Must not have been added to a different backend.""" + if adapter.backend is not None: + if adapter.backend is self: + FancyLogger.get_logger().warning( + f"attempted to add adapter {adapter.name} with type {adapter.adapter_type} to the same backend {adapter.backend}" + ) + return + else: + raise Exception( + f"adapter {adapter.name} with type {adapter.adapter_type} has already been added to backend {adapter.backend}" + ) - if self.get_alora(adapter.qualified_name) is not None: + if self._added_adapters.get(adapter.qualified_name) is not None: FancyLogger.get_logger().warning( - f"Client code attempted to add {adapter.name} but {adapter.name} was already added to {self.__class__}. The backend is refusing to do this, because adapter loading is not idempotent." + f"Client code attempted to add {adapter.name} with type {adapter.adapter_type} but {adapter.name} was already added to {self.__class__}. The backend is refusing to do this, because adapter loading is not idempotent." ) return None - self._model.load_adapter(path, adapter.qualified_name) - self._model.disable_adapters() - self._aloras[adapter.qualified_name] = adapter - - def add_alora(self, alora: HFAlora): - """Loads an ALora for this backend. - - Args: - alora (str): identifier for the ALora adapter - """ - assert issubclass(alora.__class__, HFAlora), ( - f"cannot add an ALora of type {alora.__class__} to model; must inherit from {HFAlora.__class__}" - ) - assert alora._backend == self, "Cannot load an ALora into the wrong backend." + base_model_name = self._hf_model_id.split("/")[1] + adapter.path = adapter.get_local_hf_path(base_model_name) + adapter.backend = self + self._added_adapters[adapter.qualified_name] = adapter - if self.get_alora(alora.name) is not None: - FancyLogger.get_logger().warning( - f"Client code attempted to add {alora.name} but {alora.name} was already added to {self.__class__}. The backend is refusing to do this, because ALora loading is not idempotent." + def load_adapter(self, adapter_qualified_name: str): + """Loads the given adapter for the backend. Must have previously been added.""" + adapter = self._added_adapters.get(adapter_qualified_name, None) + if adapter is None: + raise ValueError( + f"could not load adapter {adapter_qualified_name} for backend {self}: adapter was not previously added" ) - return None - - self._model.load_adapter(alora.path_or_model_id, alora.name) - self._model.disable_adapters() - self._aloras[alora.name] = alora - - def get_alora(self, alora_name: str) -> Alora | None: - """Returns the ALora by name, or None if that ALora isn't loaded.""" - return self._aloras.get(alora_name) - - def get_aloras(self) -> list[Alora]: - """Returns a list of all loaded ALora adapters.""" - return list(self._aloras.values()) - # endregion + try: + self._model.load_adapter(adapter.path, adapter.qualified_name) + except ValueError as e: + # If it's just that it's already loaded, ignore it. + if f"Adapter with name {adapter_qualified_name} already exists." not in str( + e + ): + raise e + # Loading an adapter activates it. We disable adapters immediately after. + # Prefer this over `.disable_adapters()`; the disable function doesn't always + # seem to work. + self._model.set_adapter([]) + self._loaded_adapters[adapter.qualified_name] = adapter -class HFAlora(Alora, abc.ABC): - """ALoras that work with the local huggingface backend.""" + def unload_adapter(self, adapter_qualified_name: str): + """Unloads the given adapter from the backend.""" + # Check if the backend knows about this adapter. + adapter = self._added_adapters.get(adapter_qualified_name, None) + if adapter is None: + FancyLogger.get_logger().info( + f"could not unload adapter {adapter_qualified_name} for backend {self}: adapter is not loaded" + ) + return - def __init__( - self, - name: str, - path_or_model_id: str, - generation_prompt: str, - backend: LocalHFBackend, - ): - """Initialize an ALora that should work with huggingface backends that support ALoras. + self._model.delete_adapter(adapter.qualified_name) - Args: - name (str): An arbitrary name/label to assign to an ALora. This is irrelevant from the alora's (huggingface) model id. - path_or_model_id (str): A local path to ALora's weights or a Huggingface model_id to an ALora. - generation_prompt (str): A prompt used to "activate" the Lora. This string goes between the pre-activation context and the aLora generate call. This needs to be provided by the entity that trained the ALora. - backend (LocalHFBackend): Mained as a pointer to the backend to which this this ALora is attached. - """ - super().__init__(name) - self.path_or_model_id = path_or_model_id - self._backend = backend - self._generation_prompt = generation_prompt - self._generation_prompt_tokens = self._backend._tokenizer( - self._generation_prompt, return_tensors="pt" - ) + # Remove the adapter from the list of loaded adapters. + del self._loaded_adapters[adapter.qualified_name] class HFProcessRewardModel(PRM, abc.ABC): diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 028631b7e..1a1c8fe7c 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -20,11 +20,12 @@ import mellea.backends.model_ids as model_ids from mellea.backends import BaseModelSubclass from mellea.backends.adapters.adapter import ( + AdapterMixin, + AdapterType, GraniteCommonAdapter, OpenAIAdapter, get_adapter_for_intrinsic, ) -from mellea.backends.aloras import AdapterMixin, Alora, AloraBackendMixin from mellea.backends.formatter import Formatter, FormatterBackend, TemplateFormatter from mellea.backends.model_ids import ModelIdentifier from mellea.backends.tools import ( @@ -171,7 +172,8 @@ def __init__( # Call once to create an async_client and populate the cache. _ = self._async_client - # ALoras that have been loaded for this model. + # Adapters can be made know to the backend (added) and + # loaded / active. self._added_adapters: dict[str, OpenAIAdapter] = {} self._loaded_adapters: dict[str, OpenAIAdapter] = {} @@ -293,14 +295,13 @@ def generate_from_context( assert ctx.is_chat_context, NotImplementedError( "The Openai backend only supports chat-like contexts." ) - mot = self.generate_from_chat_context( + return self.generate_from_chat_context( action, ctx, _format=format, model_options=model_options, tool_calls=tool_calls, ) - return mot, ctx.add(action).add(mot) def generate_from_chat_context( self, @@ -311,86 +312,59 @@ def generate_from_chat_context( | None = None, # Type[BaseModelSubclass] is a class object of a subclass of BaseModel model_options: dict | None = None, tool_calls: bool = False, - ) -> ModelOutputThunk: + ) -> tuple[ModelOutputThunk, Context]: """Generates a new completion from the provided Context using this backend's `Formatter`.""" - if issubclass(type(action), Requirement): - # The general rule is that we reroute to the alora if it exists. - reroute_to_alora = self.get_alora("constraint") is not None - # However, there are some exceptions: - if not self.default_to_constraint_checking_alora: - reroute_to_alora = False + # Requirements can be automatically rerouted to a requirement adapter. + if isinstance(action, Requirement): + # See docs/dev/requirement_aLoRA_rerouting.md + reroute_to_alora = self.default_to_constraint_checking_alora + adapter_name = "requirement_check" + + if isinstance(action, ALoraRequirement): + reroute_to_alora = True + adapter_name = action.intrinsic_name + alora_action = action + else: + assert action.description is not None, ( + "must have a description when generating from a requirement" + ) + alora_action = ALoraRequirement(action.description, adapter_name) + + # Check if a requirement_check (or AloraRequirement specified) adapter exists. + alora_req_adapter = get_adapter_for_intrinsic( + adapter_name, [AdapterType.ALORA], self._added_adapters + ) + if alora_req_adapter is None: + # Log a warning if using an AloraRequirement but no adapter fit. + if reroute_to_alora: + FancyLogger.get_logger().warning( + f"attempted to use an AloraRequirement but backend {self} doesn't have the specified adapter added {adapter_name}; defaulting to regular generation" + ) + if issubclass(type(action), LLMaJRequirement): reroute_to_alora = False - if issubclass(type(action), ALoraRequirement): - reroute_to_alora = True + if reroute_to_alora: - return self._generate_from_chat_context_alora( - action, ctx, _format=_format, model_options=model_options + # Keep the alora requirement handling separate for now. + mot = self._generate_from_intrinsic( + alora_action, ctx, model_options=model_options ) + return mot, ctx.add(alora_action).add(mot) elif isinstance(action, Intrinsic): - return self._generate_from_intrinsic( + mot = self._generate_from_intrinsic( action, ctx, model_options=model_options ) + return mot, ctx.add(action).add(mot) - return self._generate_from_chat_context_standard( + mot = self._generate_from_chat_context_standard( action, ctx, _format=_format, model_options=model_options, tool_calls=tool_calls, ) - - def _generate_from_chat_context_alora( - self, - action: Component | CBlock, - ctx: Context, - *, - _format: type[BaseModelSubclass] - | None = None, # Type[BaseModelSubclass] is a class object of a subclass of BaseModel - model_options: dict | None = None, - ) -> ModelOutputThunk: - match action: - case ALoraRequirement(): - alora_for_this_request = ( - self.get_alora("constraint") - if action.alora is None - else action.alora - ) - case _: - alora_for_this_request = self.get_alora("constraint") - assert alora_for_this_request is not None, ( - "This code block should not execute unless there is a 'constraint' alora loaded." - ) - - # Construct the linearized context. This is very similar to normal generation. - linearized_ctx = ctx.view_for_generation() - assert linearized_ctx is not None and len(linearized_ctx) > 1 - msgs = self.formatter.to_chat_messages(linearized_ctx) - user_message, assistant_message = msgs[-2].content, msgs[-1].content - assert alora_for_this_request is not None - assert type(user_message) is str - assert type(assistant_message) is str - assert _format is None, "Structured outputs are not supported by ALoRAs." - - model_opts = self._simplify_and_merge(model_options, is_chat_context=True) - - alora_output = alora_for_this_request.generate_using_strings( - input=user_message, - response=assistant_message, - constraint=action.description, # type: ignore - stream=model_opts.get(ModelOption.STREAM, False), - ) - - # The alora function doesn't set up all the fields. - alora_output._context = linearized_ctx - alora_output._action = action - alora_output._model_options = model_options - - # TODO: Figure out what info we want to populate for aloras here. - alora_output._generate_log = GenerateLog() - - return alora_output + return mot, ctx.add(action).add(mot) def _generate_from_intrinsic( self, action: Intrinsic, ctx: Context, *, model_options: dict | None = None @@ -406,19 +380,29 @@ def _generate_from_intrinsic( linearized_context = ctx.view_for_generation() assert linearized_context is not None, ( "Cannot generate from a non-linear context in a FormatterBackend." - ) # TODO: JAL. Log a warning if this is empty?... + ) + if len(linearized_context) == 0: + FancyLogger.get_logger().warning( + f"generating with an intrinsic when the context is empty; this is typically incorrect: {action}" + ) # Convert our linearized context into a sequence of chat messages. Template formatters have a standard way of doing this. messages: list[Message] = self.formatter.to_chat_messages(linearized_context) conversation: list[dict] = [] - # TODO: JAL. Need to handle system prompts? Confirm what granite common does. system_prompt = model_opts.get(ModelOption.SYSTEM_PROMPT, "") if system_prompt != "": conversation.append({"role": "system", "content": system_prompt}) conversation.extend([self.message_to_openai_message(m) for m in messages]) - docs = self.messages_to_docs(messages) # TODO: JAL. Improve docs interface. + docs = self.messages_to_docs(messages) + + if model_opts.get(ModelOption.STREAM, None) is not None: + # Intrinsics don't support streaming because of their post-processing step. + FancyLogger.get_logger().warning( + "intrinsics cannot use streaming; removing model option" + ) + del model_opts[ModelOption.STREAM] adapter = get_adapter_for_intrinsic( action.intrinsic_name, action.adapter_types, self._added_adapters @@ -460,26 +444,72 @@ def _generate_from_intrinsic( } rewritten = rewriter.transform(request_json, **action.intrinsic_kwargs) - # TODO: JAL. Move this comment to hugging face. - # TODO: Handle caching here. Need to see if granite_common gives us any indication - # of what messages have changed. We will also have to support caching at a - # Component / message level. - - # TODO: JAL. This needs to be made async with processing and post_processing. - # chat_response: Coroutine[ - # Any, Any, openai.AsyncStream[Completion] | Completion - # ] = self._async_client.chat.completions.create( - # **rewritten.model_dump() - # ) + self.load_adapter(adapter.qualified_name) - chat_response = self._client.chat.completions.create(**rewritten.model_dump()) + chat_response: Coroutine[Any, Any, ChatCompletion] = ( + self._async_client.chat.completions.create(**rewritten.model_dump()) + ) + + output = ModelOutputThunk(None) + output._context = linearized_context + output._action = action + output._model_options = model_opts + output._meta["granite_common_chat_response"] = rewritten + + # Add another step to the processing function. + async def granite_common_processing( + mot: ModelOutputThunk, + chunk: ChatCompletion, + rewritten: ChatCompletion, + result_processor: granite_common.IntrinsicsResultProcessor, + ): + res = result_processor.transform(chunk, rewritten) # type: ignore + + # processing expects a ChatCompletion object. Granite common differs slightly from this. Re-create the necessary object. + full_res = ChatCompletion( + id=chunk.id, + choices=[], + created=chunk.created, + model=chunk.model, + usage=chunk.usage, + object="chat.completion", + ) + + # Set the choices here so that pydantic validation doesn't error out. + full_res.choices = res.choices # type: ignore + + return await self.processing(mot, full_res) + + output._process = functools.partial( + granite_common_processing, + rewritten=rewritten, # type: ignore + result_processor=result_processor, + ) + + output._post_process = functools.partial( + self.post_processing, + tools={}, + conversation=conversation, + thinking=None, + seed=model_opts.get(ModelOption.SEED, None), + _format=None, + ) - processed_chat_completion = result_processor.transform(chat_response, rewritten) + try: + # To support lazy computation, will need to remove this create_task and store just the unexecuted coroutine. + # We can also support synchronous calls by adding a flag and changing this ._generate function. + + # This function should always be called from a running event loop so we don't have to worry about + # scheduling the task to a specific event loop here. + output._generate = asyncio.create_task( + send_to_queue(chat_response, output._async_queue) + ) + output._generate_type = GenerateType.ASYNC + except RuntimeError as e: + # Most likely cause is running this function without an event loop present + raise e - # TODO: JAL. Put into ModelOutputThunk, parse, etc... - mot = ModelOutputThunk(processed_chat_completion.choices[0].message.content) - mot._generate_log = GenerateLog() - return mot + return output @staticmethod def message_to_openai_message(msg: Message): @@ -518,12 +548,13 @@ def message_to_openai_message(msg: Message): @staticmethod def messages_to_docs(msgs: list[Message]) -> list[dict[str, str]]: + """Extracts the docs from a list of messages.""" docs: list[Document] = [] for message in msgs: if message._docs is not None: docs.extend(message._docs) - # TODO: We can add doc_ids here for vllm. + # TODO: We can add doc_ids here for vllm if needed. json_docs: list[dict[str, str]] = [] for doc in docs: json_doc = {"text": doc.text} @@ -820,7 +851,7 @@ def _generate_from_raw( return results def add_adapter(self, adapter: OpenAIAdapter): - # Gets the path / downloads files... ie does setup. + """Adds the given adapter to the backend. Must not have been added to a different backend.""" if adapter.backend is not None: if adapter.backend is self: FancyLogger.get_logger().warning( @@ -832,22 +863,21 @@ def add_adapter(self, adapter: OpenAIAdapter): f"adapter {adapter.name} with type {adapter.adapter_type} has already been added to backend {adapter.backend}" ) - base_model_name = self._hf_model_id.split("/")[-1] - adapter.path = adapter.get_open_ai_path( - base_model_name, server_type=self._server_type - ) - if self._added_adapters.get(adapter.qualified_name, None) is not None: FancyLogger.get_logger().warning( f"Client code attempted to add {adapter.name} with type {adapter.adapter_type} but it was already added to {self.__class__}. This attempt to add the adapter will be ignored." ) return None + base_model_name = self._hf_model_id.split("/")[-1] + adapter.path = adapter.get_open_ai_path( + base_model_name, server_type=self._server_type + ) + adapter.backend = self self._added_adapters[adapter.qualified_name] = adapter def load_adapter(self, adapter_qualified_name: str): - # Actually loads the adapter... - + """Loads the given adapter for the backend. Must have previously been added.""" adapter = self._added_adapters.get(adapter_qualified_name, None) if adapter is None: raise ValueError( @@ -884,8 +914,7 @@ def load_adapter(self, adapter_qualified_name: str): self._loaded_adapters[adapter.qualified_name] = adapter def unload_adapter(self, adapter_qualified_name: str): - # Unloads the adapter... - + """Unloads the given adapter from the backend.""" # Check if the backend knows about this adapter. adapter = self._added_adapters.get(adapter_qualified_name, None) if adapter is None: @@ -921,81 +950,9 @@ def unload_adapter(self, adapter_qualified_name: str): f"error unloading adapter {adapter_qualified_name}: {url}: status {response.status_code} {response.text}" ) - # Remove the alora from the list of loaded adapters. + # Remove the adapter from the list of loaded adapters. del self._loaded_adapters[adapter.qualified_name] - # TODO: JAL. Remove these functions. - # def add_alora(self, alora: "OpenAIAlora"): - # """Loads an ALora for this backend. - - # Args: - # alora (str): identifier for the ALora adapter - # """ - # assert issubclass(alora.__class__, OpenAIAlora), ( - # f"cannot add an ALora of type {alora.__class__} to model; must inherit from {OpenAIAlora.__class__}" - # ) - # assert alora._backend == self, "Cannot load an ALora into the wrong backend." - - # if self.get_alora(alora.name) is not None: - # FancyLogger.get_logger().warning( - # f"Client code attempted to add {alora.name} but {alora.name} was already added to {self.__class__}. The backend is refusing to do this, because ALora loading is not idempotent." - # ) - # return None - - # assert _server_type(self._base_url) == _ServerType.LOCALHOST, ( - # "alora is supported only for locally running vllm instances" - # ) - - # # TODO: JAL. Make sure all hf model ids fit this way. - # # base_model_name = self._hf_model_id.split("/")[1] - # # TODO: JAL. change snapshot path to this...? - # # snapshot_path = granite_common.intrinsics.util.obtain_lora( - # # alora.name, base_model_name, alora=True - # # ) - # snapshot_path = f"/u/jakelorocco/eiger-user-folder/mellea-public/test/backends/test_openai_vllm/rag-intrinsics-lib/{alora.name}/alora/granite-3.3-8b-instruct" - - # # https://docs.vllm.ai/en/stable/features/lora.html#using-api-endpoints - # # curl -X POST http://localhost:8000/v1/load_lora_adapter \ - # # -H "Content-Type: application/json" \ - # # -d '{ - # # "lora_name": "sql_adapter", - # # "lora_path": "/path/to/sql-lora-adapter" - # # }' - - # url = f"{self._base_url}/load_lora_adapter" - # response = requests.post( - # url, - # json={"lora_name": alora.name, "lora_path": snapshot_path}, - # headers={"Content-Type": "application/json"}, - # ) - - # # TODO: Add a check here for the lora/alora already being loaded. - # # TODO: See what happens if you try load the lora with the same name... - # # TODO: If the alora isn't loaded, we should raise an error or at least not - # # add it to the list of aloras. - # match response.status_code: - # case 200: - # FancyLogger.get_logger().info( - # f"{url}: status {response.status_code} {response.text}" - # ) - # self._aloras[alora.name] = alora - # case _: - # FancyLogger.get_logger().error( - # f"{url}: status {response.status_code} {response.text}" - # ) - - # self._aloras[alora.name] = alora - - # return None - - # def get_alora(self, alora_name: str) -> Alora | None: - # """Returns the ALora by name, or None if that ALora isn't loaded.""" - # return self._aloras.get(alora_name) - - # def get_aloras(self) -> list[Alora]: - # """Returns a list of all loaded ALora adapters.""" - # return list(self._aloras.values()) - def apply_chat_template(self, chat: list[dict[str, str]]): """Apply the chat template for the model, if such a model is available (e.g., when it can deduce the huggingface model id).""" from transformers import AutoTokenizer @@ -1014,23 +971,3 @@ def apply_chat_template(self, chat: list[dict[str, str]]): ) return self._tokenizer.apply_chat_template(chat, tokenize=False) - - -class OpenAIAlora(Alora, abc.ABC): - """ALoras that work with OpenAI backend.""" - - def __init__( - self, name: str, path: str, generation_prompt: str, backend: OpenAIBackend - ): - """Initialize an ALora that should work with OpenAI backends that support ALoras. - - Args: - name (str): An arbitrary name/label to assign to an ALora. This is irrelevant from the alora's (huggingface) model id. - path (str): A local path to ALora's weights or a Huggingface model_id to an ALora. - generation_prompt (str): A prompt used to "activate" the Lora. This string goes between the pre-activation context and the aLora generate call. This needs to be provided by the entity that trained the ALora. - backend (OpenAIBackend): Mained as a pointer to the backend to which this this ALora is attached. - """ - super().__init__(name) - self.path = path - self._backend = backend - self._generation_prompt = generation_prompt diff --git a/mellea/stdlib/base.py b/mellea/stdlib/base.py index e6f185904..cc546a4bc 100644 --- a/mellea/stdlib/base.py +++ b/mellea/stdlib/base.py @@ -114,14 +114,6 @@ def __repr__(self): return f"ImageBlock({self._value}, {self._meta.__repr__()})" -# TODO: JAL. Make this a CBlock? it should be a CBlock with two fields... -# TODO: JAL. Add support for passing in docs as model options? -class Document: - def __init__(self, text: str, title: str | None = None): - self.text = text - self.title = title - - @runtime_checkable class Component(Protocol): """A `Component` is a composite data structure that is intended to be represented to an LLM.""" @@ -157,6 +149,32 @@ def get_images_from_component(c: Component) -> None | list[ImageBlock]: return None +# TODO: Add support for passing in docs as model options. +class Document(Component): + """Documents should typically be used in a Message object.""" + + def __init__(self, text: str, title: str | None = None): + """Create a document object. Should typically be used as a list in the `_docs` field of Message.""" + self.text = text + self.title = title + + def parts(self) -> list[Component | CBlock]: + """The set of all the constituent parts of the `Component`.""" + raise NotImplementedError("parts isn't implemented by default") + + def format_for_llm(self) -> str: + """Formats the `Document` into a string. + + Returns: a string + """ + doc = "" + if self.title is not None: + doc += f"'{self.title}': " + doc += f"{self.text}" + + return doc + + class GenerateType(enum.Enum): """Used to track what functions can be used to extract a value from a ModelOutputThunk.""" diff --git a/mellea/stdlib/chat.py b/mellea/stdlib/chat.py index c4b445a77..574e6fa6f 100644 --- a/mellea/stdlib/chat.py +++ b/mellea/stdlib/chat.py @@ -63,7 +63,12 @@ def format_for_llm(self) -> TemplateRepresentation: """ return TemplateRepresentation( obj=self, - args={"role": self.role, "content": self.content, "images": self.images}, + args={ + "role": self.role, + "content": self.content, + "images": self.images, + "documents": self._docs, + }, template_order=["*", "Message"], ) @@ -72,7 +77,11 @@ def __str__(self): images = [] if self.images is not None: images = [f"{i[:20]}..." for i in self.images] - return f'mellea.Message(role="{self.role}", content="{self.content}", images="{images}")' + + docs = [] + if self._docs is not None: + docs = [f"{doc.format_for_llm()[:10]}..." for doc in self._docs] + return f'mellea.Message(role="{self.role}", content="{self.content}", images="{images}", documents="{docs}")' class ToolMessage(Message): diff --git a/mellea/stdlib/intrinsics/intrinsic.py b/mellea/stdlib/intrinsics/intrinsic.py index c12befd48..1554f8360 100644 --- a/mellea/stdlib/intrinsics/intrinsic.py +++ b/mellea/stdlib/intrinsics/intrinsic.py @@ -1,3 +1,5 @@ +"""Module for Intrinsics.""" + import pathlib from copy import copy from typing import cast @@ -50,11 +52,10 @@ def parts(self) -> list[Component | CBlock]: raise NotImplementedError("parts isn't implemented by default") def format_for_llm(self) -> TemplateRepresentation | str: - """Formats the `Intrinsic` into a `TemplateRepresentation` or string. + """`Intrinsic` doesn't implement `format_for_default`. Formats the `Intrinsic` into a `TemplateRepresentation` or string. Returns: a `TemplateRepresentation` or string - TODO: JAL. see if the base intrinsic should implement this. - ideally, this would be the intrinsic's instruction directive with the args populated - however, the rewriter's transform function handles that... """ - raise NotImplementedError("format_for_llm isn't implemented by default") + raise NotImplementedError( + "`Intrinsic` doesn't implement format_for_llm by default. You should only use an `Intrinsic` as the action and not as a part of the context." + ) diff --git a/mellea/stdlib/requirement.py b/mellea/stdlib/requirement.py index f10a3aafd..3168e7f45 100644 --- a/mellea/stdlib/requirement.py +++ b/mellea/stdlib/requirement.py @@ -1,13 +1,14 @@ """Requirements are a special type of Component used as input to the "validate" step in Instruct/Validate/Repair design patterns.""" import inspect +import json import re from collections.abc import Callable from copy import copy from typing import Any, overload from mellea.backends import Backend, BaseModelSubclass -from mellea.backends.aloras import Alora +from mellea.backends.adapters.adapter import AdapterType from mellea.helpers.fancy_logger import FancyLogger from mellea.stdlib.base import ( CBlock, @@ -17,6 +18,7 @@ ModelOutputThunk, TemplateRepresentation, ) +from mellea.stdlib.intrinsics.intrinsic import Intrinsic def default_output_to_bool(x: CBlock | str) -> bool: @@ -176,19 +178,54 @@ class LLMaJRequirement(Requirement): use_aloras: bool = False -class ALoraRequirement(Requirement): +def requirement_check_to_bool(x: CBlock | str) -> bool: + """Checks if a given output should be marked converted to `True`. + + By default, the requirement check alora outputs: `{"requirement_likelihood": 0.0}`. + True if >.5 + """ + output = str(x) + req_dict: dict[str, Any] = json.loads(output) + + likelihood = req_dict.get("requirement_likelihood", None) + if likelihood is None: + FancyLogger.get_logger().warning( + f"could not get value from alora requirement output; looking for `requirement_likelihood` in {req_dict}" + ) + return False + + if likelihood > 0.5: + return True + + return False + + +class ALoraRequirement(Requirement, Intrinsic): """A requirement that always uses an (possibly specified) ALora. If an exception is thrown during the ALora execution path, `mellea` will fall back to LLMaJ. But that is the only case where LLMaJ will be used.""" - def __init__(self, description: str, alora: Alora | None = None): + def __init__(self, description: str, intrinsic_name: str | None = None): """A requirement that is validated by an ALora. Args: description: See `Requirement.__init__` - alora: if None, the ALora with name "constraint" will be used. + intrinsic_name: the name of the intrinsic; must match the adapter """ - super().__init__(description, validation_fn=None) + # TODO: We may want to actually do the validation_fn here so that we can set the score. + super().__init__( + description, validation_fn=None, output_to_bool=requirement_check_to_bool + ) self.use_aloras: bool = True - self.alora = alora + + if intrinsic_name is None: + intrinsic_name = "requirement_check" + + self.intrinsic_name = intrinsic_name + self.adapter_types = [AdapterType.ALORA] + + @property + def intrinsic_kwargs(self): + """An AloraRequirement's intrinsic kwarg is always the requirement's description.""" + return {"requirement": f"{self.description}"} class ScorerRequirement(Requirement): diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index 3f40d4cd0..59869ccf9 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -4,7 +4,7 @@ from typing_extensions import Annotated from mellea import MelleaSession -from mellea.backends.aloras.huggingface.granite_aloras import add_granite_aloras +from mellea.backends.adapters.adapter import GraniteCommonAdapter from mellea.backends.cache import SimpleLRUCache from mellea.backends.formatter import TemplateFormatter from mellea.backends.huggingface import LocalHFBackend @@ -23,11 +23,11 @@ def backend(): """Shared HuggingFace backend for all tests in this module.""" backend = LocalHFBackend( - model_id="ibm-granite/granite-3.2-8b-instruct", + model_id="ibm-granite/granite-3.3-8b-instruct", formatter=TemplateFormatter(model_id="ibm-granite/granite-4.0-tiny-preview"), cache=SimpleLRUCache(5), ) - add_granite_aloras(backend) + backend.add_adapter(GraniteCommonAdapter("requirement_check")) return backend @@ -46,26 +46,6 @@ def test_system_prompt(session): ) print(result) -@pytest.mark.qualitative -async def test_constraint_alora(session, backend): - answer = session.instruct( - "Corporate wants you to find the difference between these two strings: aaaaaaaaaa aaaaabaaaa. Be concise and don't write code to answer the question.", - model_options={ - ModelOption.MAX_NEW_TOKENS: 300 - }, # Until aloras get a bit better, try not to abruptly end generation. - ) - - alora_output = backend.get_aloras()[ - 0 - ].generate_using_strings( - input="Find the difference between these two strings: aaaaaaaaaa aaaaabaaaa", - response=str(answer), - constraint="The answer mention that there is a b in the middle of one of the strings but not the other.", - force_yn=False, # make sure that the alora naturally output Y and N without constrained generation - ) - await alora_output.avalue() - assert alora_output.value in ["Y", "N"], alora_output - @pytest.mark.qualitative def test_constraint_lora_with_requirement(session, backend): answer = session.instruct( @@ -80,7 +60,7 @@ def test_constraint_lora_with_requirement(session, backend): assert len(validation_outputs) == 1 val_result = validation_outputs[0] assert isinstance(val_result, ValidationResult) - assert str(val_result.reason) in ["Y", "N"] + assert "requirement_likelihood" in str(val_result.reason) @pytest.mark.qualitative @@ -113,7 +93,7 @@ def test_constraint_lora_override_does_not_override_alora(session, backend): assert len(validation_outputs) == 1 val_result = validation_outputs[0] assert isinstance(val_result, ValidationResult) - assert str(val_result.reason) in ["Y", "N"] + assert "requirement_likelihood" in str(val_result.reason) backend.default_to_constraint_checking_alora = True @@ -132,6 +112,7 @@ def test_llmaj_req_does_not_use_alora(session, backend): val_result = validation_outputs[0] assert isinstance(val_result, ValidationResult) assert str(val_result.reason) not in ["Y", "N"] + assert "requirement_likelihood" not in str(val_result.reason) @pytest.mark.qualitative def test_instruct(session): diff --git a/test/backends/test_huggingface_tools.py b/test/backends/test_huggingface_tools.py index f78898ecd..df59eae81 100644 --- a/test/backends/test_huggingface_tools.py +++ b/test/backends/test_huggingface_tools.py @@ -4,7 +4,6 @@ import mellea.backends.model_ids as model_ids from mellea import MelleaSession -from mellea.backends.aloras.huggingface.granite_aloras import add_granite_aloras from mellea.backends.cache import SimpleLRUCache from mellea.backends.formatter import TemplateFormatter from mellea.backends.huggingface import LocalHFBackend diff --git a/test/backends/test_openai_vllm/test_openai_vllm.py b/test/backends/test_openai_vllm/test_openai_vllm.py index 505aa1ea2..dafcd1f7e 100644 --- a/test/backends/test_openai_vllm/test_openai_vllm.py +++ b/test/backends/test_openai_vllm/test_openai_vllm.py @@ -1,18 +1,17 @@ # test/rits_backend_tests/test_openai_integration.py from mellea import MelleaSession +from mellea.backends.adapters.adapter import GraniteCommonAdapter from mellea.stdlib.base import CBlock, ModelOutputThunk, ChatContext from mellea.backends.openai import OpenAIBackend -from mellea.backends.aloras.openai.granite_aloras import add_granite_aloras from mellea.stdlib.requirement import Requirement, ALoraRequirement, LLMaJRequirement, req from mellea.backends.formatter import TemplateFormatter -from mellea.backends.types import ModelOption +from mellea.backends.types import _ServerType, ModelOption import pydantic from typing_extensions import Annotated import pytest import os - # The vllm tests are disabled by default, because we need a test environment with the vLLM server running. # We use an env var VLLM_TESTS_ENABLED to enable these tests. # to run the tests, do this: VLLM_TESTS_ENABLED="1' pytest test_openai_vllm.py @@ -131,14 +130,14 @@ class Answer(pydantic.BaseModel): class TestOpenAIALoraStuff: backend = OpenAIBackend( - model_id="ibm-granite/granite-3.2-8b-instruct", + model_id="ibm-granite/granite-3.3-8b-instruct", formatter=TemplateFormatter(model_id="ibm-granite/granite-4.0-tiny-preview"), base_url="http://localhost:8000/v1", api_key="EMPTY", ) - m = MelleaSession(backend, ctx=ChatContext()) - add_granite_aloras(backend) + backend.add_adapter(GraniteCommonAdapter("requirement_check")) + m = MelleaSession(backend, ctx=ChatContext()) def test_system_prompt(self): self.m.reset() result = self.m.chat( @@ -147,21 +146,6 @@ def test_system_prompt(self): ) print(result) - @pytest.mark.xfail - def test_constraint_alora(self): - self.m.reset() - answer = self.m.instruct( - "Corporate wants you to find the difference between these two strings: aaaaaaaaaa aaaaabaaaa" - ) - alora_output = self.backend.get_aloras()[0].generate_using_strings( - input="Find the difference between these two strings: aaaaaaaaaa aaaaabaaaa", - response=str(answer), - constraint="The answer mention that there is a b in the middle of one of the strings but not the other.", - force_yn=False, # make sure that the alora naturally output Y and N without constrained generation - ) - assert alora_output in ["Y", "N"], alora_output - self.m.reset() - def test_constraint_lora_with_requirement(self): self.m.reset() answer = self.m.instruct( @@ -172,7 +156,7 @@ def test_constraint_lora_with_requirement(self): ) assert len(validation_outputs) == 1 val_result = validation_outputs[0] - assert str(val_result.reason) in ["Y", "N"] + assert "requirement_likelihood" in str(val_result.reason) self.m.reset() def test_constraint_lora_override(self): @@ -203,7 +187,7 @@ def test_constraint_lora_override_does_not_override_alora(self): ) assert len(validation_outputs) == 1 non_alora_output = validation_outputs[0] - assert str(non_alora_output.reason) in ["Y", "N"] + assert "requirement_likelihood" in str(non_alora_output.reason) self.backend.default_to_constraint_checking_alora = True self.m.reset() diff --git a/test/stdlib_basics/test_base.py b/test/stdlib_basics/test_base.py index e19c6adc1..917619e0e 100644 --- a/test/stdlib_basics/test_base.py +++ b/test/stdlib_basics/test_base.py @@ -26,6 +26,5 @@ def format_for_llm(self) -> str: c = _ClosuredComponent() assert len(c.parts()) == 0 - if __name__ == "__main__": pytest.main([__file__]) diff --git a/test/stdlib_basics/test_chat.py b/test/stdlib_basics/test_chat.py new file mode 100644 index 000000000..3ae911b97 --- /dev/null +++ b/test/stdlib_basics/test_chat.py @@ -0,0 +1,25 @@ +import pytest +from mellea.backends.openai import OpenAIBackend +from mellea.stdlib.base import Document +from mellea.stdlib.chat import Message + +def test_message_with_docs(): + doc = Document("I'm text!", "Im a title!") + msg = Message("user", "hello", documents=[doc]) + + assert msg._docs is not None + assert doc in msg._docs + + docs = OpenAIBackend.messages_to_docs([msg]) + assert len(docs) == 1 + assert docs[0]["text"] == doc.text + assert docs[0]["title"] == doc.title + + assert "Im a titl..." in str(msg) + + tr = msg.format_for_llm() + assert tr.args["documents"] + + +if __name__ == "__main__": + pytest.main([__file__]) From 78364adfd0fdc2fe034468763296925550148ed8 Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Mon, 3 Nov 2025 09:35:13 -0500 Subject: [PATCH 11/38] feat: add docs --- docs/dev/intrinsics_and_adapters.md | 37 +++++++++++++++++++++++++ docs/dev/requirement_aLoRA_rerouting.md | 9 +++--- 2 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 docs/dev/intrinsics_and_adapters.md diff --git a/docs/dev/intrinsics_and_adapters.md b/docs/dev/intrinsics_and_adapters.md new file mode 100644 index 000000000..3cbeccbc7 --- /dev/null +++ b/docs/dev/intrinsics_and_adapters.md @@ -0,0 +1,37 @@ +# Intrinsics and Adapters + +## Basics +In Mellea, intrinsics are a type of Component that signals one or more of the following to a backend: +- a special adapter must be used for generation +- the input/output for generation must be transformed in a particular way +- the model options must be modified in a particular way + +These changes only happen when the intrinsic is the "action" of the request. Intrinsics should usually not be used as an item in the context of generation (in fact, by default, Intrinsics have no string representation). + +These changes are specified by the Adapter that corresponds to a given Intrinsic. Matching happens based on the adapter name and type. + +## Parts of an Intrinsic +Intrinsics specify: +- an adapter name (ie requirement_check) +- types of adapters suitable to be used (ie alora) +- any kwargs necessary (ie a requirement like "make sure the last user message is...") + +## Parts of an Adapter +Adapters specify: +- compatible backends +- adapter type +- functions for getting a path to load them + +## Using Intrinsics +Mellea Intrinsics currently utilize the granite-common package for loading adapters and formatting input/outputs (https://github.com/ibm-granite/granite-common). This means Mellea only allows intrinsics/adapters that follow this pattern. + +## Needed Future Work +### Custom Adapters / Intrinsics +Mellea should support custom intrinsic / adapter implementations. To do this: +- make backend `_generate_from_intrinsic` functions generic and utilize only common adapter functions +- adapters must specify a transformation function that encapsulates the input/output modifications necessary for their generation requests + +### Concurrency Checks +Some backends (currently only LocalHFBackend) that allow adapters to be loaded, cannot independently utilize these adapters without impacting other generation requests. + +These backends should support a generation lock that ensures requests are only performed when the correct set of adapters (or no adapters) are active. diff --git a/docs/dev/requirement_aLoRA_rerouting.md b/docs/dev/requirement_aLoRA_rerouting.md index f7001df0a..d21fb7770 100644 --- a/docs/dev/requirement_aLoRA_rerouting.md +++ b/docs/dev/requirement_aLoRA_rerouting.md @@ -14,14 +14,14 @@ The actual rule is slightly more complicated. ## The Actual Rule -If a `Requirement` is validated using a backend that could either use a `constraint` aLoRA or perform an LLMaJ prompt on the underlying model, then the aLoRA is used for validation, even if the `backend.generate_from_context` method is called instead of the `alora.generate_from_strings` method. +If a `Requirement` is validated using a backend that could either use a `requirement_check` aLoRA or perform an LLMaJ prompt on the underlying model, then the aLoRA is used for validation, even if the `backend.generate_from_context` method is called instead of the `backend._generate_from_intrinsic` method. There are three exceptions to this rule: 1. `Backend.default_to_constraint_checking_alora` is set to `False` (this parameter defaults to `True`). 2. The `Requirement` has a more specific subtype that indicates a more specific intent (`LLMaJRequirement`). 3. The `ALoRA` requirement checker throws an exception. -There is an exception (or disambiguation) to the first exception: If the user provides an `ALoRARequirement`, then the `backend.generate_from_context` call is rerouted to the constraint checking LoRA, regardless of the value of `deault_to_constraint_checking_alora`. +There is an exception (or disambiguation) to the first exception: If the user provides an `ALoRARequirement`, then the `backend.generate_from_context` call is rerouted to the constraint checking LoRA, regardless of the value of `default_to_constraint_checking_alora`. ## Decision Rationale @@ -33,12 +33,13 @@ Suppose that the user creates a backend and then adds a generic constraint check ```python from mellea import start_session -from mellea.backends.aloras.granite_aloras import add_granite_aloras from mellea.stdlib.requirement import Requirement m = start_session( "huggingface.LocalHFBackend:ibm-granite/granite-3.2-8b-instruct") -add_granite_aloras(m) # This will load the Constraint checint aLoRA. + +# By default, the AloraRequirement uses a GraniteCommonAdapter with "requirement_check". +m.backend.add_adapter(GraniteCommonAdapter("requirement_check")) m.instruct( "Corporate wants you to find the difference between these two strings:\n\naaa\naba") From 17b7f8ebf9252edb3c7ed1e7661db05e6e7149cb Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Mon, 3 Nov 2025 10:02:43 -0500 Subject: [PATCH 12/38] feat: add specific intrinsic and adapter tests --- docs/examples/intrinsics/intrinsics.py | 45 +++++++++++++++++++ mellea/backends/huggingface.py | 2 +- mellea/backends/openai.py | 2 +- mellea/stdlib/intrinsics/intrinsic.py | 2 - .../intrinsics-data/answerability.yaml | 25 +++++++++++ test/backends/test_adapters/test_adapter.py | 18 ++++++++ test/backends/test_huggingface.py | 15 +++++++ .../test_openai_vllm/test_openai_vllm.py | 16 +++++++ 8 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 docs/examples/intrinsics/intrinsics.py create mode 100644 test/backends/test_adapters/intrinsics-data/answerability.yaml create mode 100644 test/backends/test_adapters/test_adapter.py diff --git a/docs/examples/intrinsics/intrinsics.py b/docs/examples/intrinsics/intrinsics.py new file mode 100644 index 000000000..98fd46af7 --- /dev/null +++ b/docs/examples/intrinsics/intrinsics.py @@ -0,0 +1,45 @@ +from mellea.backends.openai import OpenAIBackend, _ServerType +from mellea.backends.adapters.adapter import AdapterType, GraniteCommonAdapter +from mellea.stdlib.base import ChatContext, ModelOutputThunk +from mellea.stdlib.chat import Message +import mellea.stdlib.funcs as mfuncs +from mellea.stdlib.intrinsics.intrinsic import Intrinsic + +# Create the Adapter. GraniteCommonAdapter's default to ALORAs. +req_adapter = GraniteCommonAdapter("requirement_check") + +# Create the backend. Assumes a locally running VLLM server. +backend = OpenAIBackend( + model_id="ibm-granite/granite-3.3-8b-instruct", + base_url="http://0.0.0.0:8000/v1", + api_key="EMPTY", +) + +# If using a remote VLLM server, utilize the `test/backends/test_openai_vllm/serve.sh` +# script with `export VLLM_DOWNLOAD_RAG_INTRINSICS=True`. This will download the granite_common +# adapters on the server. +backend._server_type = _ServerType.REMOTE_VLLM + +# Add the adapter to the backend. +backend.add_adapter(req_adapter) + +ctx = ChatContext() +ctx = ctx.add(Message("user", "Hi, can you help me?")) +ctx = ctx.add(Message("assistant", "Hello; yes! What can I help with?")) + +# Generate from an intrinsic with the same name as the adapter. By default, it will look for +# ALORA and then LORA adapters. +out, new_ctx = mfuncs.act( + Intrinsic( + "requirement_check", + intrinsic_kwargs={"requirement": "The assistant is helpful."}, + ), + ctx, + backend, +) + +# Print the output. The requirement_check adapter has a specific output format: +print(out) # {"requirement_likelihood": 1.0} + +# The AloraRequirement uses this adapter. It automatically parses that output +# when validating the output. diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index 660d381df..3622a6cf9 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -845,7 +845,7 @@ def load_adapter(self, adapter_qualified_name: str): def unload_adapter(self, adapter_qualified_name: str): """Unloads the given adapter from the backend.""" # Check if the backend knows about this adapter. - adapter = self._added_adapters.get(adapter_qualified_name, None) + adapter = self._loaded_adapters.get(adapter_qualified_name, None) if adapter is None: FancyLogger.get_logger().info( f"could not unload adapter {adapter_qualified_name} for backend {self}: adapter is not loaded" diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 1a1c8fe7c..45c85cfdb 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -916,7 +916,7 @@ def load_adapter(self, adapter_qualified_name: str): def unload_adapter(self, adapter_qualified_name: str): """Unloads the given adapter from the backend.""" # Check if the backend knows about this adapter. - adapter = self._added_adapters.get(adapter_qualified_name, None) + adapter = self._loaded_adapters.get(adapter_qualified_name, None) if adapter is None: FancyLogger.get_logger().info( f"could not unload adapter {adapter_qualified_name} for backend {self}: adapter is not loaded" diff --git a/mellea/stdlib/intrinsics/intrinsic.py b/mellea/stdlib/intrinsics/intrinsic.py index 1554f8360..2d6e95985 100644 --- a/mellea/stdlib/intrinsics/intrinsic.py +++ b/mellea/stdlib/intrinsics/intrinsic.py @@ -4,8 +4,6 @@ from copy import copy from typing import cast -import granite_common - from mellea.backends.adapters.adapter import AdapterType from mellea.stdlib.base import CBlock, Component, TemplateRepresentation diff --git a/test/backends/test_adapters/intrinsics-data/answerability.yaml b/test/backends/test_adapters/intrinsics-data/answerability.yaml new file mode 100644 index 000000000..2b72cd180 --- /dev/null +++ b/test/backends/test_adapters/intrinsics-data/answerability.yaml @@ -0,0 +1,25 @@ +# Model name string, or null to use whatever is provided in the chat completion request. +model: ~ +# JSON schema of the model's output +response_format: | + { + "type": "string", + "enum": ["answerable", "unanswerable"] + } +transformations: + # Convert categorical answer to continuous value by decoding logprobs + - type: likelihood + categories_to_values: + "answerable": 1.0 + "unanswerable": 0.0 + input_path: [] + # Convert scalar value to a record for consistency with other intrinsics + - type: nest + input_path: [] + field_name: "answerability_likelihood" +instruction: ~ +parameters: + # "unanswerable" can be 6 tokens at high temperatures + max_completion_tokens: 6 +# No sentence boundary detection +sentence_boundaries: ~ diff --git a/test/backends/test_adapters/test_adapter.py b/test/backends/test_adapters/test_adapter.py new file mode 100644 index 000000000..b7ae746ea --- /dev/null +++ b/test/backends/test_adapters/test_adapter.py @@ -0,0 +1,18 @@ +import pathlib +import pytest + +from mellea.backends.adapters.adapter import GraniteCommonAdapter + +# The backend tests handle most of the adapter testing. Do a basic test here +# to make sure init and config loading work. +def test_adapter_init(): + dir_file = pathlib.Path(__file__).parent.joinpath("intrinsics-data") + answerability_file = f"{dir_file}/answerability.yaml" + + adapter = GraniteCommonAdapter("answerability", config_file=answerability_file) + + assert adapter.config is not None + assert adapter.config["parameters"]["max_completion_tokens"] == 6 + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index 59869ccf9..6a096aace 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -38,6 +38,21 @@ def session(backend): yield session session.reset() +def test_adapters(backend): + assert len(backend._added_adapters.items()) > 0 + + adapter = backend._added_adapters["requirement_check_alora"] + backend.load_adapter(adapter.qualified_name) + assert adapter.qualified_name in backend._loaded_adapters + + # Ensure you can load the same adapter twice. + backend.load_adapter(adapter.qualified_name) + + # Ensure you can unload an adapter. + backend.unload_adapter(adapter.qualified_name) + backend.unload_adapter(adapter.qualified_name) + assert adapter.qualified_name not in backend._loaded_adapters + @pytest.mark.qualitative def test_system_prompt(session): result = session.chat( diff --git a/test/backends/test_openai_vllm/test_openai_vllm.py b/test/backends/test_openai_vllm/test_openai_vllm.py index dafcd1f7e..2ae84490f 100644 --- a/test/backends/test_openai_vllm/test_openai_vllm.py +++ b/test/backends/test_openai_vllm/test_openai_vllm.py @@ -138,6 +138,22 @@ class TestOpenAIALoraStuff: backend.add_adapter(GraniteCommonAdapter("requirement_check")) m = MelleaSession(backend, ctx=ChatContext()) + + def test_adapters(self): + assert len(self.backend._added_adapters.items()) > 0 + + adapter = self.backend._added_adapters["requirement_check_alora"] + self.backend.load_adapter(adapter.qualified_name) + assert adapter.qualified_name in self.backend._loaded_adapters + + # Ensure you can load the same adapter twice. + self.backend.load_adapter(adapter.qualified_name) + + # Ensure you can unload an adapter. + self.backend.unload_adapter(adapter.qualified_name) + self.backend.unload_adapter(adapter.qualified_name) + assert adapter.qualified_name not in self.backend._loaded_adapters + def test_system_prompt(self): self.m.reset() result = self.m.chat( From 51e9c5019a7470622e59f37cb56c19921e9b6011 Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Mon, 3 Nov 2025 10:28:42 -0500 Subject: [PATCH 13/38] fix: add note that only gcommon adapters/intrinsics are supported --- docs/dev/intrinsics_and_adapters.md | 1 + mellea/backends/huggingface.py | 4 +++- mellea/backends/openai.py | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/dev/intrinsics_and_adapters.md b/docs/dev/intrinsics_and_adapters.md index 3cbeccbc7..a41c3e5bc 100644 --- a/docs/dev/intrinsics_and_adapters.md +++ b/docs/dev/intrinsics_and_adapters.md @@ -1,4 +1,5 @@ # Intrinsics and Adapters +Note: Mellea currently only supports GraniteCommonAdapters and Intrinsics. ## Basics In Mellea, intrinsics are a type of Component that signals one or more of the following to a backend: diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index 3622a6cf9..7c20e093a 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -278,7 +278,9 @@ def _generate_from_intrinsic( # TODO: Code below this point is mostly specific to RagIntrinsics (and granite_common). # It should be refactored into a specific adapter.transform() function. - assert isinstance(adapter, GraniteCommonAdapter) + assert isinstance(adapter, GraniteCommonAdapter), ( + "currently Mellea only supports GraniteCommonAdapters and Intrinsics" + ) intrinsic_config = adapter.config if intrinsic_config is None: diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 45c85cfdb..84b75f86a 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -414,7 +414,9 @@ def _generate_from_intrinsic( # TODO: Code below this point is mostly specific to RagIntrinsics (and granite_common). # It should be refactored into a specific adapter.transform() function. - assert isinstance(adapter, GraniteCommonAdapter) + assert isinstance(adapter, GraniteCommonAdapter), ( + "currently Mellea only supports GraniteCommonAdapters and Intrinsics" + ) intrinsic_config = adapter.config if intrinsic_config is None: From ff91cbd05550c84b94ae9c9173dfc69ead4808cd Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Mon, 3 Nov 2025 11:56:57 -0500 Subject: [PATCH 14/38] fix: add docstrings for types --- mellea/backends/types.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mellea/backends/types.py b/mellea/backends/types.py index 3c4e21a70..89f03851e 100644 --- a/mellea/backends/types.py +++ b/mellea/backends/types.py @@ -114,6 +114,8 @@ def merge_model_options( class _ServerType(Enum): + """Different types of servers that might be relevant for a backend.""" + UNKNOWN = 0 LOCALHOST = 1 OPENAI = 2 @@ -122,6 +124,7 @@ class _ServerType(Enum): def _server_type(url: str) -> _ServerType: + """Find a server type based on the url.""" try: parsed = urlparse(url) hostname = parsed.hostname From 9145f56bc7e3a1962e3c7670e00d4f135f173deb Mon Sep 17 00:00:00 2001 From: jakelorocco Date: Mon, 3 Nov 2025 12:27:03 -0500 Subject: [PATCH 15/38] fix: update model in openai vllm test --- test/backends/test_openai_vllm/test_openai_vllm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/backends/test_openai_vllm/test_openai_vllm.py b/test/backends/test_openai_vllm/test_openai_vllm.py index 2ae84490f..cdfe997eb 100644 --- a/test/backends/test_openai_vllm/test_openai_vllm.py +++ b/test/backends/test_openai_vllm/test_openai_vllm.py @@ -28,8 +28,8 @@ class TestOpenAIBackend: backend = OpenAIBackend( - model_id="ibm-granite/granite-3.2-8b-instruct", - formatter=TemplateFormatter(model_id="ibm-granite/granite-3.2-8b-instruct"), + model_id="ibm-granite/granite-3.3-8b-instruct", + formatter=TemplateFormatter(model_id="ibm-granite/granite-3.3-8b-instruct"), base_url="http://0.0.0.0:8000/v1", api_key="EMPTY", ) From 929c20176bb089f497b93d134c62bd8adf6f94da Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Thu, 6 Nov 2025 10:48:01 -0800 Subject: [PATCH 16/38] Add entry point for answerability Signed-off-by: Fred Reiss --- mellea/backends/adapters/adapter.py | 100 ++++++++++++------ mellea/backends/huggingface.py | 36 ++++--- mellea/backends/openai.py | 43 ++++---- mellea/stdlib/intrinsics/intrinsic.py | 7 +- mellea/stdlib/intrinsics/rag.py | 96 +++++++++++++++++ mellea/stdlib/requirement.py | 4 + test/backends/test_adapters/test_adapter.py | 4 +- test/backends/test_huggingface.py | 3 +- test/stdlib_intrinsics/test_rag/test_rag.py | 84 +++++++++++++++ .../testdata/input_json/answerability.json | 20 ++++ 10 files changed, 330 insertions(+), 67 deletions(-) create mode 100644 mellea/stdlib/intrinsics/rag.py create mode 100644 test/stdlib_intrinsics/test_rag/test_rag.py create mode 100644 test/stdlib_intrinsics/test_rag/testdata/input_json/answerability.json diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 7b7236614..52002bb43 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -6,6 +6,7 @@ from typing import Any, TypeVar import granite_common +import yaml from litellm import cast from mellea.backends import Backend @@ -81,52 +82,70 @@ class GraniteCommonAdapter(OpenAIAdapter, LocalHFAdapter): def __init__( self, - name: str, + repo_id: str, + intrinsic_name: str, adapter_type: AdapterType = AdapterType.ALORA, config_file: str | pathlib.Path | None = None, config_dict: dict | None = None, base_model_name: str | None = None, ): - """An adapter that can be added to either an `OpenAIBackend` or a `LocalHFBackend`. Most rag-lib-intrinsics support lora or alora adapter types. + """Entry point for creating GraniteCommonAdapter objects. + + An adapter that can be added to either an `OpenAIBackend` or a `LocalHFBackend`. + Most intrinsics support LoRA or aLoRA adapter types. Args: - name: name of the adapter; when referencing this adapter, use adapter.qualified_name + repo_id: Name of Hugging Face Hub repository containing the adapters that + implement the intrinsic; for example, + "generative-computing/rag-intrinsics-lib" + intrinsic_name: name of the intrinsic; the local name of the loaded adapter + that implements this intrinsic will be adapter.qualified_name adapter_type: enum describing what type of adapter it is (ie LORA / ALORA) config_file: optional; file for defining the intrinsic / transformations config_dict: optional; dict for defining the intrinsic / transformations - base_model_name: optional; if provided with no config_file/config_dict, will be used to lookup the granite_common config for this adapter + base_model_name: optional; if provided with no config_file/config_dict, + will be used to look up the granite_common config for this adapter """ - assert adapter_type == AdapterType.ALORA or adapter_type == AdapterType.LORA, ( + assert adapter_type in (AdapterType.ALORA, AdapterType.LORA), ( f"{adapter_type} not supported" ) - super().__init__(name, adapter_type) + super().__init__(f"{repo_id}_{intrinsic_name}", adapter_type) + self.repo_id = repo_id + self.intrinsic_name = intrinsic_name self.base_model_name = base_model_name # If any of the optional params are specified, attempt to set up the # config for the intrinsic here. - config: dict | None = None - if config_file is not None or config_dict is not None: - config = granite_common.intrinsics.util.make_config_dict( - config_file=config_file, config_dict=config_dict + if config_file and config_dict: + raise ValueError( + f"Conflicting values for config_file and config_dict " + f"parameters provided. Values were {config_file=} " + f"and {config_dict=}" ) - config = cast( - dict, config - ) # Can remove if util function gets exported properly. - - if config is None and self.base_model_name is not None: - is_alora = True if self.adapter_type == AdapterType.ALORA else False - io_yaml_file = granite_common.intrinsics.util.obtain_io_yaml( - self.name, self.base_model_name, alora=is_alora + if config_file is None and config_dict is None and base_model_name is None: + raise ValueError( + "At least one of [config_file, config_dict, base_model_name] " + "must be provided." ) - config = granite_common.intrinsics.util.make_config_dict( - config_file=io_yaml_file + if config_file is None and config_dict is None: + is_alora = self.adapter_type == AdapterType.ALORA + config_file = granite_common.intrinsics.util.obtain_io_yaml( + self.intrinsic_name, + self.base_model_name, + alora=is_alora, + repo_id=repo_id, ) - config = cast( - dict, config - ) # Can remove if util function gets exported properly. - - self.config: dict | None = config + if config_file: + with open(config_file, encoding="utf-8") as f: + config_dict = yaml.safe_load(f) + if not isinstance(config_dict, dict): + raise ValueError( + f"YAML file {config_file} does not evaluate to a " + f"dictionary when parsed." + ) + assert config_dict is not None # Code above should initialize this variable + self.config: dict = config_dict def get_open_ai_path( self, @@ -137,9 +156,12 @@ def get_open_ai_path( """Returns the path needed to load the adapter. Args: - base_model_name: the base model; typically the last part of the huggingface model id like "granite-3.3-8b-instruct" - server_type: the server type (ie LOCALHOST / OPENAI); usually the backend has information on this - remote_path: optional; used only if the server_type is REMOTE_VLLM; base path at which to find the adapter + base_model_name: the base model; typically the last part of the huggingface + model id like "granite-3.3-8b-instruct" + server_type: the server type (ie LOCALHOST / OPENAI); usually the backend + has information on this + remote_path: optional; used only if the server_type is REMOTE_VLLM; base + path at which to find the adapter """ if server_type == _ServerType.LOCALHOST: path = self.download_and_get_path(base_model_name) @@ -174,12 +196,16 @@ def download_and_get_path(self, base_model_name: str) -> str: is_alora = self.adapter_type == AdapterType.ALORA return str( granite_common.intrinsics.util.obtain_lora( - self.name, base_model_name, alora=is_alora + self.intrinsic_name, + base_model_name, + alora=is_alora, + repo_id=self.repo_id, ) ) def get_path_on_remote(self, base_model_name: str, base_path: str) -> str: """Assumes the files have already been downloaded on the remote server.""" + # TODO: This will break when we switch to the new repo!!! return f"./{base_path}/{self.name}/{self.adapter_type.value}/{base_model_name}" @@ -187,6 +213,7 @@ def get_path_on_remote(self, base_model_name: str, base_path: str) -> str: def get_adapter_for_intrinsic( + repo_id: str, intrinsic_name: str, intrinsic_adapter_types: list[AdapterType], available_adapters: dict[str, T], @@ -194,6 +221,8 @@ def get_adapter_for_intrinsic( """Finds an adapter from a dict of available adapters based on the intrinsic name and its allowed adapter types. Args: + repo_id: Name of Hugging Face Hub repository containing the adapters that + implement the intrinsic intrinsic_name: the name of the intrinsic, like "answerability" intrinsic_adapter_types: the adapter types allowed for this intrinsic, like ALORA / LORA available_adapters: the available adapters to choose from; maps adapter.qualified_name to the Adapter @@ -203,8 +232,8 @@ def get_adapter_for_intrinsic( """ adapter = None for adapter_type in intrinsic_adapter_types: - qualified_name = intrinsic_name + "_" + adapter_type.value - adapter = available_adapters.get(qualified_name, None) + qualified_name = f"{repo_id}_{intrinsic_name}_{adapter_type.value}" + adapter = available_adapters.get(qualified_name) if adapter is not None: break @@ -222,3 +251,12 @@ def load_adapter(self, adapter_qualified_name: str): def unload_adapter(self, adapter_qualified_name: str): """Unloads the given adapter from the backend.""" + + def list_adapters(self) -> list[str]: + """Lists the adapters added via add_adapter(). + + :returns: list of adapter names that are currently registered with this backend + """ + raise NotImplementedError( + f"Backend type {type(self)} does not implement list_adapters() API call." + ) diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index 7c20e093a..cb51cb3f9 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -65,7 +65,12 @@ ) from mellea.stdlib.chat import Message from mellea.stdlib.intrinsics.intrinsic import Intrinsic -from mellea.stdlib.requirement import ALoraRequirement, LLMaJRequirement, Requirement +from mellea.stdlib.requirement import ( + REQUIREMENT_REPO_ID, + ALoraRequirement, + LLMaJRequirement, + Requirement, +) assert outlines, "outlines needs to be present to make outlines_core work" @@ -204,7 +209,10 @@ def generate_from_context( # Check if a requirement_check (or AloraRequirement specified) adapter exists. alora_req_adapter = get_adapter_for_intrinsic( - adapter_name, [AdapterType.ALORA], self._added_adapters + REQUIREMENT_REPO_ID, + adapter_name, + [AdapterType.ALORA], + self._added_adapters, ) if alora_req_adapter is None: # Log a warning if using an AloraRequirement but no adapter fit. @@ -269,7 +277,10 @@ def _generate_from_intrinsic( del model_options[ModelOption.STREAM] adapter = get_adapter_for_intrinsic( - action.intrinsic_name, action.adapter_types, self._added_adapters + action.repo_id, + action.intrinsic_name, + action.adapter_types, + self._added_adapters, ) if adapter is None: raise ValueError( @@ -283,17 +294,7 @@ def _generate_from_intrinsic( ) intrinsic_config = adapter.config - if intrinsic_config is None: - # If the adapter wasn't initialized with a config, grab one here based off the backend's model. - intrinsic_config_file = granite_common.intrinsics.util.obtain_io_yaml( - action.intrinsic_name, self._hf_model_id.split("/")[-1] - ) - intrinsic_config = granite_common.intrinsics.util.make_config_dict( - config_file=intrinsic_config_file - ) - intrinsic_config = cast( - dict, intrinsic_config - ) # TODO: Can remove if util function gets exported properly. + assert intrinsic_config is not None rewriter = granite_common.IntrinsicsRewriter( config_dict=intrinsic_config, model_name=adapter.name @@ -859,6 +860,13 @@ def unload_adapter(self, adapter_qualified_name: str): # Remove the adapter from the list of loaded adapters. del self._loaded_adapters[adapter.qualified_name] + def list_adapters(self) -> list[str]: + """Lists the adapters added via add_adapter(). + + :returns: list of adapter names that are currently registered with this backend + """ + return list(self._loaded_adapters.keys()) + class HFProcessRewardModel(PRM, abc.ABC): """A Process Reward Model that works with a huggingface backend.""" diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 84b75f86a..c20418770 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -55,7 +55,12 @@ ) from mellea.stdlib.chat import Message from mellea.stdlib.intrinsics.intrinsic import Intrinsic -from mellea.stdlib.requirement import ALoraRequirement, LLMaJRequirement, Requirement +from mellea.stdlib.requirement import ( + REQUIREMENT_REPO_ID, + ALoraRequirement, + LLMaJRequirement, + Requirement, +) if TYPE_CHECKING: from transformers.tokenization_utils import PreTrainedTokenizer @@ -332,7 +337,10 @@ def generate_from_chat_context( # Check if a requirement_check (or AloraRequirement specified) adapter exists. alora_req_adapter = get_adapter_for_intrinsic( - adapter_name, [AdapterType.ALORA], self._added_adapters + REQUIREMENT_REPO_ID, + adapter_name, + [AdapterType.ALORA], + self._added_adapters, ) if alora_req_adapter is None: # Log a warning if using an AloraRequirement but no adapter fit. @@ -405,7 +413,10 @@ def _generate_from_intrinsic( del model_opts[ModelOption.STREAM] adapter = get_adapter_for_intrinsic( - action.intrinsic_name, action.adapter_types, self._added_adapters + action.repo_id, + action.intrinsic_name, + action.adapter_types, + self._added_adapters, ) if adapter is None: raise ValueError( @@ -417,25 +428,12 @@ def _generate_from_intrinsic( assert isinstance(adapter, GraniteCommonAdapter), ( "currently Mellea only supports GraniteCommonAdapters and Intrinsics" ) - - intrinsic_config = adapter.config - if intrinsic_config is None: - # If the adapter wasn't initialized with a config, grab one here based off the backend's model. - intrinsic_config_file = granite_common.intrinsics.util.obtain_io_yaml( - action.intrinsic_name, self._hf_model_id.split("/")[-1] - ) - intrinsic_config = granite_common.intrinsics.util.make_config_dict( - config_file=intrinsic_config_file - ) - intrinsic_config = cast( - dict, intrinsic_config - ) # TODO: Can remove if util function gets exported properly. - + assert adapter.config is not None rewriter = granite_common.IntrinsicsRewriter( - config_dict=intrinsic_config, model_name=adapter.qualified_name + config_dict=adapter.config, model_name=adapter.qualified_name ) result_processor = granite_common.IntrinsicsResultProcessor( - config_dict=intrinsic_config + config_dict=adapter.config ) # Convert our conversation into a proper chat completions dict. @@ -955,6 +953,13 @@ def unload_adapter(self, adapter_qualified_name: str): # Remove the adapter from the list of loaded adapters. del self._loaded_adapters[adapter.qualified_name] + def list_adapters(self) -> list[str]: + """Lists the adapters added via add_adapter(). + + :returns: list of adapter names that are currently registered with this backend + """ + return list(self._loaded_adapters.keys()) + def apply_chat_template(self, chat: list[dict[str, str]]): """Apply the chat template for the model, if such a model is available (e.g., when it can deduce the huggingface model id).""" from transformers import AutoTokenizer diff --git a/mellea/stdlib/intrinsics/intrinsic.py b/mellea/stdlib/intrinsics/intrinsic.py index 2d6e95985..f17b1f762 100644 --- a/mellea/stdlib/intrinsics/intrinsic.py +++ b/mellea/stdlib/intrinsics/intrinsic.py @@ -13,6 +13,7 @@ class Intrinsic(Component): def __init__( self, + repo_id: str, intrinsic_name: str, intrinsic_kwargs: dict | None = None, adapter_types: list[AdapterType] = [AdapterType.ALORA, AdapterType.LORA], @@ -28,10 +29,14 @@ def __init__( An intrinsic component should correspond to a loaded adapter. Args: - intrinsic_name: the name of the intrinsic; must match the adapter + repo_id: name of Hugging Face Hub repository containing adapters that + implement the intrinsic + intrinsic_name: the name of the intrinsic; must match the name of the + associated adapters within the target repository intrinsic_kwargs: some intrinsics require kwargs when utilizing them; provide them here adapter_types: list of adapter types that can be used for this intrinsic """ + self.repo_id = repo_id self.intrinsic_name = intrinsic_name # Copy the list so that this intrinsic has its own list that can be modified independently. diff --git a/mellea/stdlib/intrinsics/rag.py b/mellea/stdlib/intrinsics/rag.py new file mode 100644 index 000000000..1e4025912 --- /dev/null +++ b/mellea/stdlib/intrinsics/rag.py @@ -0,0 +1,96 @@ +"""Intrinsic functions related to retrieval-augmented generation.""" + +import collections.abc +import json + +import granite_common + +import mellea.stdlib.funcs +from mellea.backends import Backend +from mellea.backends.adapters.adapter import ( + AdapterMixin, + AdapterType, + GraniteCommonAdapter, +) +from mellea.stdlib.base import ChatContext, Document +from mellea.stdlib.chat import Message +from mellea.stdlib.intrinsics.intrinsic import Intrinsic + +# Temporary value +RAG_REPO = "ibm-granite/rag-intrinsics-lib" +# RAG_REPO = "generative-computing/rag-intrinsics-lib" + +# Mapping from name to repository, adapter_type +_CATALOG = {"answerability": (RAG_REPO, AdapterType.LORA)} +ANSWERABILITY_MODEL_NAME = "answerability" + + +def check_answerability( + context: ChatContext, + question: str, + docs: collections.abc.Iterable[Document], + backend, # Can't put type annotations here because linter complains +) -> float: + """Test a user's question for answerability. + + Intrinsic function that checks whether the question in the last user turn of a + chat can be answered by a provided set of RAG documents. + + :param context: Chat context containing the conversation thus far + :param question: Question that the user has posed in response to the last turn in + ``context``. + :param docs: A set of documents retrieved that may or may not answer the indicated + question. + :param backend: Backend instance that supports adding the LoRA or aLoRA adapters + for answerability checks + + :return: Answerability score as a floating-point value from 0 to 1. + """ + # Adapter needs to be present in the backend before it can be invoked. + # The current APIs require us to create the Adapter object in order to determine + # whether we need to create the Adapter object. + intrinsic_name = "answerability" + repo_id, adapter_type = _CATALOG[intrinsic_name] + base_model_name = backend.model_id + if base_model_name is None: + raise ValueError("Backend has no model ID") + adapter = GraniteCommonAdapter( + repo_id, intrinsic_name, adapter_type, base_model_name=base_model_name + ) + if adapter.qualified_name not in backend.list_adapters(): + backend.add_adapter(adapter) + + # Append the user's question and the documents to the existing conversation. + # This operation creates a copy of the immutable context. + context = context.add(Message("user", question, documents=list(docs))) + + # Create the AST node for the action we wish to perform. + intrinsic = Intrinsic(repo_id, intrinsic_name, adapter_types=[adapter_type]) + + # Execute the AST node. + model_output_thunk, _ = mellea.stdlib.funcs.act( + intrinsic, + context, + backend, + # No rejection sampling, please + strategy=None, + ) + + # act() can return a future. Don't know how to handle one from non-async code. + assert model_output_thunk.is_computed() + + # Output of an Intrinsic action is the string representation of the output of the + # intrinsic. Parse the string. + result_str = model_output_thunk.value + if result_str is None: + raise ValueError("Model output is None.") + result_json = json.loads(result_str) + if ( + not isinstance(result_json, dict) + or "answerability_likelihood" not in result_json + ): + raise ValueError( + f"Unexpected low-level result format from {intrinsic_name} " + f"adapter: '{result_str}'" + ) + return result_json["answerability_likelihood"] diff --git a/mellea/stdlib/requirement.py b/mellea/stdlib/requirement.py index 3168e7f45..fcc7f7d7f 100644 --- a/mellea/stdlib/requirement.py +++ b/mellea/stdlib/requirement.py @@ -20,6 +20,10 @@ ) from mellea.stdlib.intrinsics.intrinsic import Intrinsic +REQUIREMENT_REPO_ID = "ibm-granite/rag-intrinsics-lib" +"""Hard-coded repository on Hugging Face Hub where Mellea keeps its requirement +intrinsic's aLoRA adapter.""" + def default_output_to_bool(x: CBlock | str) -> bool: """Checks if a given output should be marked converted to `True`. diff --git a/test/backends/test_adapters/test_adapter.py b/test/backends/test_adapters/test_adapter.py index b7ae746ea..1d03ba00c 100644 --- a/test/backends/test_adapters/test_adapter.py +++ b/test/backends/test_adapters/test_adapter.py @@ -9,7 +9,9 @@ def test_adapter_init(): dir_file = pathlib.Path(__file__).parent.joinpath("intrinsics-data") answerability_file = f"{dir_file}/answerability.yaml" - adapter = GraniteCommonAdapter("answerability", config_file=answerability_file) + adapter = GraniteCommonAdapter( + "ibm-granite/rag-intrinsics-lib", + "answerability", config_file=answerability_file) assert adapter.config is not None assert adapter.config["parameters"]["max_completion_tokens"] == 6 diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index 6a096aace..8cd759ebf 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -27,7 +27,8 @@ def backend(): formatter=TemplateFormatter(model_id="ibm-granite/granite-4.0-tiny-preview"), cache=SimpleLRUCache(5), ) - backend.add_adapter(GraniteCommonAdapter("requirement_check")) + backend.add_adapter(GraniteCommonAdapter("ibm-granite/rag-intrinsics-lib", + "requirement_check")) return backend diff --git a/test/stdlib_intrinsics/test_rag/test_rag.py b/test/stdlib_intrinsics/test_rag/test_rag.py new file mode 100644 index 000000000..1562dcedf --- /dev/null +++ b/test/stdlib_intrinsics/test_rag/test_rag.py @@ -0,0 +1,84 @@ +"""Tests of the code in ``mellea.stdlib.intrinsics.rag``""" + +import gc +import os +import json +import pathlib + +import pytest +import torch + +from mellea.backends.formatter import TemplateFormatter +from mellea.backends.huggingface import LocalHFBackend +from mellea.stdlib.base import ChatContext, Document +from mellea.stdlib.chat import Message +from mellea.stdlib.intrinsics import rag + + +DATA_ROOT = pathlib.Path(os.path.dirname(__file__)) / "testdata" +"""Location of data files for the tests in this file.""" + +BASE_MODEL = "ibm-granite/granite-3.3-2b-instruct" + + +@pytest.fixture +def backend(): + """Backend used by the tests in this file.""" + + backend = LocalHFBackend( + model_id=BASE_MODEL, formatter=TemplateFormatter(model_id=BASE_MODEL) + ) + yield backend + + # Begin cleanup code + del backend + gc.collect() # Force a collection of the newest generation + gc.collect() + gc.collect() # Hopefully force a collection of the oldest generation + torch.cuda.empty_cache() + + +def _read_input_json(file_name: str): + """Shared code for reading data stored in JSON files and converting to Mellea + types.""" + with open(DATA_ROOT / "input_json" / file_name, encoding="utf-8") as f: + json_data = json.load(f) + + # Data is assumed to be an OpenAI chat completion request. Convert to Mellea format. + context = ChatContext() + for m in json_data["messages"][:-1]: + context.add(Message(m["role"], m["content"])) + + # Store the user turn at the end of the messages list separately so that tests can + # play it back. + next_user_turn = json_data["messages"][-1]["content"] + + documents = [] + if "extra_body" in json_data and "documents" in json_data["extra_body"]: + for d in json_data["extra_body"]["documents"]: + documents.append( + Document( + text=d["text"], + # Mellea doesn't have a doc_id, but OpenAI requires it. Compromise + # by converting the doc_id to a title. + title=d["doc_id"], + ) + ) + return context, next_user_turn, documents + + +def test_answerability(backend): + """Verify that the answerability intrinsic functions properly.""" + context, next_user_turn, documents = _read_input_json("answerability.json") + + # First call triggers LoRA adapter loading + result = rag.check_answerability(context, next_user_turn, documents, backend) + assert pytest.approx(result) == 1.0 + + # Second call hits a different code path from the first one + result = rag.check_answerability(context, next_user_turn, documents, backend) + assert pytest.approx(result) == 1.0 + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/test/stdlib_intrinsics/test_rag/testdata/input_json/answerability.json b/test/stdlib_intrinsics/test_rag/testdata/input_json/answerability.json new file mode 100644 index 000000000..8b4df2c7c --- /dev/null +++ b/test/stdlib_intrinsics/test_rag/testdata/input_json/answerability.json @@ -0,0 +1,20 @@ +{ + "messages": [ + { + "role": "assistant", + "content": "Hello there, how can I help you?" + }, + { + "content": "What is the square root of 4?", + "role": "user" + } + ], + "extra_body": { + "documents": [ + { + "doc_id": "1", + "text": "The square root of 4 is 2." + } + ] + } +} \ No newline at end of file From f5c1311793575d4a9f162d0be7996a4fdf2b189e Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Fri, 7 Nov 2025 20:11:15 -0800 Subject: [PATCH 17/38] Rewrite, citations, and context relevance Signed-off-by: Fred Reiss --- mellea/backends/openai.py | 3 +- mellea/stdlib/base.py | 5 +- mellea/stdlib/intrinsics/rag.py | 179 ++++++++++++++---- mellea/stdlib/requirement.py | 16 +- test/backends/test_huggingface.py | 11 +- test/stdlib_intrinsics/test_rag/test_rag.py | 88 +++++++-- .../testdata/input_json/citations.json | 24 +++ .../input_json/context_relevance.json | 16 ++ .../testdata/input_json/query_rewrite.json | 29 +++ .../testdata/output_json/citations.json | 11 ++ 10 files changed, 316 insertions(+), 66 deletions(-) create mode 100644 test/stdlib_intrinsics/test_rag/testdata/input_json/citations.json create mode 100644 test/stdlib_intrinsics/test_rag/testdata/input_json/context_relevance.json create mode 100644 test/stdlib_intrinsics/test_rag/testdata/input_json/query_rewrite.json create mode 100644 test/stdlib_intrinsics/test_rag/testdata/output_json/citations.json diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index c20418770..5a04feaad 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -554,12 +554,13 @@ def messages_to_docs(msgs: list[Message]) -> list[dict[str, str]]: if message._docs is not None: docs.extend(message._docs) - # TODO: We can add doc_ids here for vllm if needed. json_docs: list[dict[str, str]] = [] for doc in docs: json_doc = {"text": doc.text} if doc.title is not None: json_doc["title"] = doc.title + if doc.doc_id is not None: + json_doc["doc_id"] = doc.doc_id json_docs.append(json_doc) return json_docs diff --git a/mellea/stdlib/base.py b/mellea/stdlib/base.py index cc546a4bc..111d44f69 100644 --- a/mellea/stdlib/base.py +++ b/mellea/stdlib/base.py @@ -153,10 +153,11 @@ def get_images_from_component(c: Component) -> None | list[ImageBlock]: class Document(Component): """Documents should typically be used in a Message object.""" - def __init__(self, text: str, title: str | None = None): + def __init__(self, text: str, title: str | None = None, doc_id: str | None = None): """Create a document object. Should typically be used as a list in the `_docs` field of Message.""" self.text = text self.title = title + self.doc_id = doc_id def parts(self) -> list[Component | CBlock]: """The set of all the constituent parts of the `Component`.""" @@ -168,6 +169,8 @@ def format_for_llm(self) -> str: Returns: a string """ doc = "" + if self.doc_id is not None: + doc += f"document ID '{self.doc_id}': " if self.title is not None: doc += f"'{self.title}': " doc += f"{self.text}" diff --git a/mellea/stdlib/intrinsics/rag.py b/mellea/stdlib/intrinsics/rag.py index 1e4025912..029346e85 100644 --- a/mellea/stdlib/intrinsics/rag.py +++ b/mellea/stdlib/intrinsics/rag.py @@ -16,40 +16,34 @@ from mellea.stdlib.chat import Message from mellea.stdlib.intrinsics.intrinsic import Intrinsic -# Temporary value RAG_REPO = "ibm-granite/rag-intrinsics-lib" # RAG_REPO = "generative-computing/rag-intrinsics-lib" -# Mapping from name to repository, adapter_type -_CATALOG = {"answerability": (RAG_REPO, AdapterType.LORA)} +# Mapping from name to . +_CATALOG = { + "answerability": (RAG_REPO, AdapterType.LORA), + "query_rewrite": (RAG_REPO, AdapterType.LORA), + "citations": (RAG_REPO, AdapterType.LORA), + "context_relevance": (RAG_REPO, AdapterType.LORA), +} ANSWERABILITY_MODEL_NAME = "answerability" -def check_answerability( - context: ChatContext, - question: str, - docs: collections.abc.Iterable[Document], - backend, # Can't put type annotations here because linter complains -) -> float: - """Test a user's question for answerability. +def _call_intrinsic( + intrinsic_name: str, context: ChatContext, backend, /, kwargs: dict | None = None +): + """Shared code for invoking intrinsics. - Intrinsic function that checks whether the question in the last user turn of a - chat can be answered by a provided set of RAG documents. - - :param context: Chat context containing the conversation thus far - :param question: Question that the user has posed in response to the last turn in - ``context``. - :param docs: A set of documents retrieved that may or may not answer the indicated - question. - :param backend: Backend instance that supports adding the LoRA or aLoRA adapters - for answerability checks - - :return: Answerability score as a floating-point value from 0 to 1. + :returns: Result of the call in JSON format. """ # Adapter needs to be present in the backend before it can be invoked. - # The current APIs require us to create the Adapter object in order to determine - # whether we need to create the Adapter object. - intrinsic_name = "answerability" + # We must create the Adapter object in order to determine whether we need to create + # the Adapter object. + if intrinsic_name not in _CATALOG: + raise ValueError( + f"Unexpected intrinsic name '{intrinsic_name}'. " + f"Should be one of {list(_CATALOG.keys())}" + ) repo_id, adapter_type = _CATALOG[intrinsic_name] base_model_name = backend.model_id if base_model_name is None: @@ -60,12 +54,10 @@ def check_answerability( if adapter.qualified_name not in backend.list_adapters(): backend.add_adapter(adapter) - # Append the user's question and the documents to the existing conversation. - # This operation creates a copy of the immutable context. - context = context.add(Message("user", question, documents=list(docs))) - # Create the AST node for the action we wish to perform. - intrinsic = Intrinsic(repo_id, intrinsic_name, adapter_types=[adapter_type]) + intrinsic = Intrinsic( + repo_id, intrinsic_name, adapter_types=[adapter_type], intrinsic_kwargs=kwargs + ) # Execute the AST node. model_output_thunk, _ = mellea.stdlib.funcs.act( @@ -85,12 +77,123 @@ def check_answerability( if result_str is None: raise ValueError("Model output is None.") result_json = json.loads(result_str) - if ( - not isinstance(result_json, dict) - or "answerability_likelihood" not in result_json - ): - raise ValueError( - f"Unexpected low-level result format from {intrinsic_name} " - f"adapter: '{result_str}'" - ) + return result_json + + +def check_answerability( + context: ChatContext, + question: str, + documents: collections.abc.Iterable[Document], + backend, # Can't put type hints here because linter complains +) -> float: + """Test a user's question for answerability. + + Intrinsic function that checks whether the question in the last user turn of a + chat can be answered by a provided set of RAG documents. + + :param context: Chat context containing the conversation thus far + :param question: Question that the user has posed in response to the last turn in + ``context``. + :param documents: A set of documents retrieved that may or may not answer the + indicated question. + :param backend: Backend instance that supports adding the LoRA or aLoRA adapters + for answerability checks + + :return: Answerability score as a floating-point value from 0 to 1. + """ + result_json = _call_intrinsic( + "answerability", + context.add(Message("user", question, documents=list(documents))), + backend, + ) return result_json["answerability_likelihood"] + + +def rewrite_question( + context: ChatContext, + question: str, + backend, # Can't put type hints here because linter complains +) -> float: + """Rewrite a user's question for retrieval. + + Intrinsic function that rewrites the question in the next user turn into a + self-contained query that can be passed to the retriever. + + :param context: Chat context containing the conversation thus far + :param question: Question that the user has posed in response to the last turn in + ``context``. + :param backend: Backend instance that supports adding the LoRA or aLoRA adapters + + :return: Rewritten version of ``question``. + """ + result_json = _call_intrinsic( + "query_rewrite", context.add(Message("user", question)), backend + ) + return result_json["rewritten_question"] + + +def find_citations( + context: ChatContext, + response: str, + documents: collections.abc.Iterable[Document], + backend, # Can't put type hints here because linter complains +) -> list[dict]: + """Find information in documents that supports an assistant response. + + Intrinsic function that finds sentences in RAG documents that support sentences + in a potential assistant response to a user question. + + :param context: Context of the dialog between user and assistant at the point where + the user has just asked a question that will be answered with RAG documents + :param response: Potential assistant response + :param documents: Documents at were used to generate ``response``. These documents + should set the ``doc_id`` field; otherwise the intrinsic will be unable to + specify which document was the source of a given citation. + :param backend: Backend that supports one of the adapters that implements this + intrinsic. + :return: List of records with the following fields: + * ``response_begin`` + * ``response_end`` + * ``response_text`` + * ``citation_doc_id`` + * ``citation_begin`` + * ``citation_end`` + * ``citation_text`` + Begin and end offsets are character offsets into their respective UTF-8 strings. + """ + result_json = _call_intrinsic( + "citations", + context.add(Message("assistant", response, documents=list(documents))), + backend, + ) + return result_json + + +def check_context_relevance( + context: ChatContext, + question: str, + document: Document, + backend, # Can't put type hints here because linter complains +) -> float: + """Test whether a document is relevant to a user's question. + + Intrinsic function that checks whether a single document contains part or all of + the answer to a user's question. Does not consider the context in which the + question was asked. + + :param context: The chat up to the point where the user asked a question. + :param question: Question that the user has posed. + :param document: A retrieved document snippet + :param backend: Backend instance that supports the adapters that implement this + intrinsic + + :return: Context relevance score as a floating-point value from 0 to 1. + """ + result_json = _call_intrinsic( + "context_relevance", + context.add(Message("user", question)), + backend, + # Target document is passed as an argument + kwargs={"document_content": document.text}, + ) + return result_json["context_relevance"] diff --git a/mellea/stdlib/requirement.py b/mellea/stdlib/requirement.py index fcc7f7d7f..6a2c817c6 100644 --- a/mellea/stdlib/requirement.py +++ b/mellea/stdlib/requirement.py @@ -14,7 +14,6 @@ CBlock, Component, Context, - GenerateLog, ModelOutputThunk, TemplateRepresentation, ) @@ -223,13 +222,14 @@ def __init__(self, description: str, intrinsic_name: str | None = None): if intrinsic_name is None: intrinsic_name = "requirement_check" - self.intrinsic_name = intrinsic_name - self.adapter_types = [AdapterType.ALORA] - - @property - def intrinsic_kwargs(self): - """An AloraRequirement's intrinsic kwarg is always the requirement's description.""" - return {"requirement": f"{self.description}"} + # Initialize the other side of the inheritance tree + Intrinsic.__init__( + self, + repo_id=REQUIREMENT_REPO_ID, + intrinsic_name=intrinsic_name, + intrinsic_kwargs={"requirement": f"{self.description}"}, + adapter_types=[AdapterType.ALORA], + ) class ScorerRequirement(Requirement): diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index 8cd759ebf..e0ed531d5 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -16,6 +16,7 @@ Requirement, ValidationResult, default_output_to_bool, + REQUIREMENT_REPO_ID ) @@ -27,8 +28,11 @@ def backend(): formatter=TemplateFormatter(model_id="ibm-granite/granite-4.0-tiny-preview"), cache=SimpleLRUCache(5), ) - backend.add_adapter(GraniteCommonAdapter("ibm-granite/rag-intrinsics-lib", - "requirement_check")) + # Assertion to assuage the wrath of the linter + assert backend.model_id is not None + backend.add_adapter(GraniteCommonAdapter(REQUIREMENT_REPO_ID, + "requirement_check", + base_model_name=backend.model_id)) return backend @@ -42,7 +46,8 @@ def session(backend): def test_adapters(backend): assert len(backend._added_adapters.items()) > 0 - adapter = backend._added_adapters["requirement_check_alora"] + expected_qualified_name = f"{REQUIREMENT_REPO_ID}_requirement_check_alora" + adapter = backend._added_adapters[expected_qualified_name] backend.load_adapter(adapter.qualified_name) assert adapter.qualified_name in backend._loaded_adapters diff --git a/test/stdlib_intrinsics/test_rag/test_rag.py b/test/stdlib_intrinsics/test_rag/test_rag.py index 1562dcedf..f9e91d403 100644 --- a/test/stdlib_intrinsics/test_rag/test_rag.py +++ b/test/stdlib_intrinsics/test_rag/test_rag.py @@ -1,37 +1,37 @@ """Tests of the code in ``mellea.stdlib.intrinsics.rag``""" import gc -import os import json +import os import pathlib import pytest import torch -from mellea.backends.formatter import TemplateFormatter from mellea.backends.huggingface import LocalHFBackend from mellea.stdlib.base import ChatContext, Document from mellea.stdlib.chat import Message from mellea.stdlib.intrinsics import rag - DATA_ROOT = pathlib.Path(os.path.dirname(__file__)) / "testdata" """Location of data files for the tests in this file.""" + BASE_MODEL = "ibm-granite/granite-3.3-2b-instruct" -@pytest.fixture -def backend(): +@pytest.fixture(name="backend") +def _backend(): """Backend used by the tests in this file.""" - backend = LocalHFBackend( - model_id=BASE_MODEL, formatter=TemplateFormatter(model_id=BASE_MODEL) + backend_ = LocalHFBackend( + model_id=BASE_MODEL ) - yield backend + yield backend_ - # Begin cleanup code - del backend + # Code after yield is cleanup code. + # Free GPU memory with extreme prejudice. + del backend_ gc.collect() # Force a collection of the newest generation gc.collect() gc.collect() # Hopefully force a collection of the oldest generation @@ -47,7 +47,7 @@ def _read_input_json(file_name: str): # Data is assumed to be an OpenAI chat completion request. Convert to Mellea format. context = ChatContext() for m in json_data["messages"][:-1]: - context.add(Message(m["role"], m["content"])) + context = context.add(Message(m["role"], m["content"])) # Store the user turn at the end of the messages list separately so that tests can # play it back. @@ -59,19 +59,31 @@ def _read_input_json(file_name: str): documents.append( Document( text=d["text"], - # Mellea doesn't have a doc_id, but OpenAI requires it. Compromise - # by converting the doc_id to a title. - title=d["doc_id"], + doc_id=d["doc_id"], ) ) return context, next_user_turn, documents +def _read_output_json(file_name: str): + """Shared code for reading canned outputs stored in JSON files and converting + to Mellea types.""" + with open(DATA_ROOT / "output_json" / file_name, encoding="utf-8") as f: + json_data = json.load(f) + + # Output is in OpenAI chat completion response format. Assume only one choice. + result_str = json_data["choices"][0]["message"]["content"] + + # Intrinsic outputs are always JSON, serialized to a string for OpenAI + # compatibility. + return json.loads(result_str) + + def test_answerability(backend): """Verify that the answerability intrinsic functions properly.""" context, next_user_turn, documents = _read_input_json("answerability.json") - # First call triggers LoRA adapter loading + # First call triggers adapter loading result = rag.check_answerability(context, next_user_turn, documents, backend) assert pytest.approx(result) == 1.0 @@ -80,5 +92,51 @@ def test_answerability(backend): assert pytest.approx(result) == 1.0 +def test_query_rewrite(backend): + """Verify that the answerability intrinsic functions properly.""" + context, next_user_turn, _ = _read_input_json("query_rewrite.json") + expected = ( + "Is Rex, the dog, more likely to get fleas because he spends a lot of " + "time outdoors?" + ) + + # First call triggers adapter loading + result = rag.rewrite_question(context, next_user_turn, backend) + assert result == expected + + # Second call hits a different code path from the first one + result = rag.rewrite_question(context, next_user_turn, backend) + assert result == expected + + +def test_citations(backend): + """Verify that the citations intrinsic functions properly.""" + context, assistant_response, docs = _read_input_json("citations.json") + expected = _read_output_json("citations.json") + + # First call triggers adapter loading + result = rag.find_citations(context, assistant_response, docs, backend) + assert result == expected + + # Second call hits a different code path from the first one + result = rag.find_citations(context, assistant_response, docs, backend) + assert result == expected + +def test_context_relevance(backend): + """Verify that the context relevance intrinsic functions properly.""" + context, question, docs = _read_input_json("context_relevance.json") + + # Context relevance can only check against a single document at a time. + document = docs[0] + + # First call triggers adapter loading + result = rag.check_context_relevance(context, question, document, backend) + assert pytest.approx(result, 2e-2) == 0.45 + + # Second call hits a different code path from the first one + result = rag.check_context_relevance(context, question, document, backend) + assert pytest.approx(result, 2e-2) == 0.45 + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/test/stdlib_intrinsics/test_rag/testdata/input_json/citations.json b/test/stdlib_intrinsics/test_rag/testdata/input_json/citations.json new file mode 100644 index 000000000..f22cfc9a6 --- /dev/null +++ b/test/stdlib_intrinsics/test_rag/testdata/input_json/citations.json @@ -0,0 +1,24 @@ +{ + "messages": [ + { + "role": "user", + "content": "How does Murdoch's expansion in Australia compare to his expansion in New Zealand?" + }, + { + "role": "assistant", + "content": "Murdoch expanded in Australia and New Zealand by acquiring and expanding local newspapers. I do not have information about his expansion in New Zealand after purchasing The Dominion." + } + ], + "extra_body": { + "documents": [ + { + "doc_id": "0", + "text": "Keith Rupert Murdoch was born on 11 March 1931 in Melbourne, Australia, the son of Sir Keith Murdoch (1885-1952) and Dame Elisabeth Murdoch (nee Greene; 1909-2012). He is of English, Irish, and Scottish ancestry. Murdoch's parents were also born in Melbourne. Keith Murdoch was a war correspondent and later a regional newspaper magnate owning two newspapers in Adelaide, South Australia, and a radio station in a faraway mining town. Following his father's death, when he was 21, Murdoch returned from Oxford to take charge of the family business News Limited, which had been established in 1923. Rupert Murdoch turned its Adelaide newspaper, The News, its main asset, into a major success. He began to direct his attention to acquisition and expansion, buying the troubled Sunday Times in Perth, Western Australia (1956) and over the next few years acquiring suburban and provincial newspapers in New South Wales, Queensland, Victoria and the Northern Territory, including the Sydney afternoon tabloid, The Daily Mirror (1960). The Economist describes Murdoch as \"inventing the modern tabloid\", as he developed a pattern for his newspapers, increasing sports and scandal coverage and adopting eye-catching headlines. Murdoch's first foray outside Australia involved the purchase of a controlling interest in the New Zealand daily The Dominion. In January 1964, while touring New Zealand with friends in a rented Morris Minor after sailing across the Tasman, Murdoch read of a takeover bid for the Wellington paper by the British-based Canadian newspaper magnate, Lord Thomson of Fleet. On the spur of the moment, he launched a counter-bid. A four-way battle for control ensued in which the 32-year-old Murdoch was ultimately successful. Later in 1964, Murdoch launched The Australian, Australia's first national daily newspaper, which was based first in Canberra and later in Sydney. In 1972, Murdoch acquired the Sydney morning tabloid The Daily Telegraph from Australian media mogul Sir Frank Packer, who later regretted selling it to him. In 1984, Murdoch was appointed Companion of the Order of Australia (AC) for services to publishing. In 1999, Murdoch significantly expanded his music holdings in Australia by acquiring the controlling share in a leading Australian independent label, Michael Gudinski's Mushroom Records; he merged that with Festival Records, and the result was Festival Mushroom Records (FMR). Both Festival and FMR were managed by Murdoch's son James Murdoch for several years." + }, + { + "doc_id": "1", + "text": "This document has nothing to do with Rupert Murdoch. This document is two sentences long." + } + ] + } +} \ No newline at end of file diff --git a/test/stdlib_intrinsics/test_rag/testdata/input_json/context_relevance.json b/test/stdlib_intrinsics/test_rag/testdata/input_json/context_relevance.json new file mode 100644 index 000000000..84cdf4e5a --- /dev/null +++ b/test/stdlib_intrinsics/test_rag/testdata/input_json/context_relevance.json @@ -0,0 +1,16 @@ +{ + "messages": [ + { + "content": "Who is the CEO of Microsoft?", + "role": "user" + } + ], + "extra_body": { + "documents": [ + { + "doc_id": "1", + "text": "Microsoft Corporation is an American multinational corporation and technology conglomerate headquartered in Redmond, Washington.[2] Founded in 1975, the company became influential in the rise of personal computers through software like Windows, and the company has since expanded to Internet services, cloud computing, video gaming and other fields. Microsoft is the largest software maker, one of the most valuable public U.S. companies,[a] and one of the most valuable brands globally." + } + ] + } +} \ No newline at end of file diff --git a/test/stdlib_intrinsics/test_rag/testdata/input_json/query_rewrite.json b/test/stdlib_intrinsics/test_rag/testdata/input_json/query_rewrite.json new file mode 100644 index 000000000..0c36933d6 --- /dev/null +++ b/test/stdlib_intrinsics/test_rag/testdata/input_json/query_rewrite.json @@ -0,0 +1,29 @@ +{ + "messages": [ + { + "role": "assistant", + "content": "Welcome to pet questions!" + }, + { + "role": "user", + "content": "I have two pets, a dog named Rex and a cat named Lucy." + }, + { + "role": "assistant", + "content": "Great, what would you like to share about them?" + }, + { + "role": "user", + "content": "Rex spends a lot of time in the backyard and outdoors, and Luna is always inside." + }, + { + "role": "assistant", + "content": "Sounds good! Rex must love exploring outside, while Lucy probably enjoys her cozy indoor life." + }, + { + "role": "user", + "content": "But is he more likely to get fleas because of that?" + } + ], + "temperature": 0.0 +} \ No newline at end of file diff --git a/test/stdlib_intrinsics/test_rag/testdata/output_json/citations.json b/test/stdlib_intrinsics/test_rag/testdata/output_json/citations.json new file mode 100644 index 000000000..804f64f43 --- /dev/null +++ b/test/stdlib_intrinsics/test_rag/testdata/output_json/citations.json @@ -0,0 +1,11 @@ +{ + "choices": [ + { + "index": 0, + "message": { + "content": "[{\"response_begin\": 0, \"response_end\": 96, \"response_text\": \"Murdoch expanded in Australia and New Zealand by acquiring and expanding local newspapers. \", \"citation_doc_id\": \"0\", \"citation_begin\": 2468, \"citation_end\": 3533, \"citation_text\": \"He began to direct his attention to acquisition and expansion, buying the troubled Sunday Times in Perth, Western Australia (1956) and over the next few years acquiring suburban and provincial newspapers in New South Wales, Queensland, Victoria and the Northern Territory, including the Sydney afternoon tabloid, The Daily Mirror (1960). \"}, {\"response_begin\": 0, \"response_end\": 96, \"response_text\": \"Murdoch expanded in Australia and New Zealand by acquiring and expanding local newspapers. \", \"citation_doc_id\": \"0\", \"citation_begin\": 4792, \"citation_end\": 6183, \"citation_text\": \"Murdoch's first foray outside Australia involved the purchase of a controlling interest in the New Zealand daily The Dominion. \"}]", + "role": "assistant" + } + } + ] +} \ No newline at end of file From bb10bd49354a5aba189ebf5a966087504317874b Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Mon, 10 Nov 2025 14:10:34 -0800 Subject: [PATCH 18/38] Add additional intrinsics Signed-off-by: Fred Reiss --- mellea/stdlib/intrinsics/rag.py | 110 +++++++++++++++++- test/stdlib_intrinsics/test_rag/test_rag.py | 53 +++++++-- .../testdata/input_json/answer_relevance.json | 13 +++ .../input_json/hallucination_detection.json | 24 ++++ .../output_json/hallucination_detection.json | 11 ++ 5 files changed, 201 insertions(+), 10 deletions(-) create mode 100644 test/stdlib_intrinsics/test_rag/testdata/input_json/answer_relevance.json create mode 100644 test/stdlib_intrinsics/test_rag/testdata/input_json/hallucination_detection.json create mode 100644 test/stdlib_intrinsics/test_rag/testdata/output_json/hallucination_detection.json diff --git a/mellea/stdlib/intrinsics/rag.py b/mellea/stdlib/intrinsics/rag.py index 029346e85..7fa412733 100644 --- a/mellea/stdlib/intrinsics/rag.py +++ b/mellea/stdlib/intrinsics/rag.py @@ -21,13 +21,27 @@ # Mapping from name to . _CATALOG = { + "answer_relevance_classifier": (RAG_REPO, AdapterType.LORA), + "answer_relevance_rewriter": (RAG_REPO, AdapterType.LORA), "answerability": (RAG_REPO, AdapterType.LORA), - "query_rewrite": (RAG_REPO, AdapterType.LORA), "citations": (RAG_REPO, AdapterType.LORA), "context_relevance": (RAG_REPO, AdapterType.LORA), + "hallucination_detection": (RAG_REPO, AdapterType.LORA), + "query_rewrite": (RAG_REPO, AdapterType.LORA), } ANSWERABILITY_MODEL_NAME = "answerability" +_ANSWER_RELEVANCE_CORRECTION_METHODS = { + "Excessive unnecessary information": "removing the excessive information from the draft response", + "Unduly restrictive": "providing answer without the unwarranted restriction, or indicating that the desired answer is not available", + "Too vague or generic": "providing more crisp and to-the-point answer, or indicating that the desired answer is not available", + "Contextual misalignment": "providing a response that answers the last user inquiry, taking into account the context of the conversation", + "Misinterpreted inquiry": "providing answer only to the correct interpretation of the inquiry, or attempting clarification if the inquiry is ambiguous or otherwise confusing, or indicating that the desired answer is not available", + "No attempt": "providing a relevant response if an inquiry should be answered, or providing a short response if the last user utterance contains no inquiry", +} +"""Prompting strings for the answer relevance rewriter. This model is a (a)LoRA adapter, +so it's important to stick to in-domain prompts.""" + def _call_intrinsic( intrinsic_name: str, context: ChatContext, backend, /, kwargs: dict | None = None @@ -94,7 +108,7 @@ def check_answerability( :param context: Chat context containing the conversation thus far :param question: Question that the user has posed in response to the last turn in ``context``. - :param documents: A set of documents retrieved that may or may not answer the + :param documents: Document snippets retrieved that may or may not answer the indicated question. :param backend: Backend instance that supports adding the LoRA or aLoRA adapters for answerability checks @@ -197,3 +211,95 @@ def check_context_relevance( kwargs={"document_content": document.text}, ) return result_json["context_relevance"] + + +def flag_hallucinated_content( + context: ChatContext, + response: str, + documents: collections.abc.Iterable[Document], + backend, # Can't put type hints here because linter complains +) -> float: + """Flag potentially-hallucinated sentences in an agent's response. + + Intrinsic function that checks whether the sentences in an agent's response to a + user question are faithful to the retrieved document snippets. Sentences that do not + align with the retrieved snippets are flagged as potential hallucinations. + + :param context: A chat log that ends with a user asking a question + :param response: The assistant's response to the user's question in the last turn + of ``context`` + :param documents: Document snippets that were used to generate ``response`` + :param backend: Backend instance that supports the adapters that implement this + intrinsic + + :return: List of records with the following fields: + * response_begin + * response_end + * response_text + * faithfulness_likelihood + * explanation + """ + result_json = _call_intrinsic( + "hallucination_detection", + context.add(Message("assistant", response, documents=list(documents))), + backend, + ) + return result_json + + +def rewrite_answer_for_relevance( + context, + response: str, + documents: collections.abc.Iterable[Document], + backend, # Can't put type hints here because linter complains + /, + rewrite_threshold: float = 0.5, +) -> str: + """Rewrite an assistant answer to improve relevance to the user's question. + + :param context: A chat log that ends with a user asking a question + :param response: The assistant's response to the user's question in the last turn + of ``context`` + :param documents: Document snippets that were used to generate ``response`` + :param backend: Backend instance that supports the adapters that implement this + intrinsic + :param rewrite_threshold: Number between 0.0 and 1.0 that determines how eagerly + to skip rewriting the assistant's answer for relevance. 0.0 means never rewrite + and 1.0 means always rewrite. + + :returns: Either the original response, or a rewritten version of the original + response. + """ + # First run the classifier to determine the likelihood of a relevant answer + # Output will have three fields: + # * answer_relevance_analysis + # * answer_relevance_category + # * answer_relevance_likelihood + result_json = _call_intrinsic( + "answer_relevance_classifier", + context.add(Message("assistant", response, documents=list(documents))), + backend, + ) + if result_json["answer_relevance_likelihood"] >= rewrite_threshold: + return response + + # If we get here, the classifier indicated a likely irrelevant response. Trigger + # rewrite. + # Rewrite needs a prompt string that is an expanded version of the classifier's + # short output. + correction_method = _ANSWER_RELEVANCE_CORRECTION_METHODS[ + result_json["answer_relevance_category"] + ] + + result_json = _call_intrinsic( + "answer_relevance_rewriter", + context.add(Message("assistant", response, documents=list(documents))), + backend, + kwargs={ + "answer_relevance_category": result_json["answer_relevance_category"], + "answer_relevance_analysis": result_json["answer_relevance_category"], + "correction_method": correction_method, + }, + ) + # Unpack boxed string + return result_json["answer_relevance_rewrite"] diff --git a/test/stdlib_intrinsics/test_rag/test_rag.py b/test/stdlib_intrinsics/test_rag/test_rag.py index f9e91d403..dc5c21ebb 100644 --- a/test/stdlib_intrinsics/test_rag/test_rag.py +++ b/test/stdlib_intrinsics/test_rag/test_rag.py @@ -23,10 +23,11 @@ @pytest.fixture(name="backend") def _backend(): """Backend used by the tests in this file.""" + + # Prevent thrashing if the default device is CPU + torch.set_num_threads(4) - backend_ = LocalHFBackend( - model_id=BASE_MODEL - ) + backend_ = LocalHFBackend(model_id=BASE_MODEL) yield backend_ # Code after yield is cleanup code. @@ -122,21 +123,57 @@ def test_citations(backend): result = rag.find_citations(context, assistant_response, docs, backend) assert result == expected + def test_context_relevance(backend): """Verify that the context relevance intrinsic functions properly.""" context, question, docs = _read_input_json("context_relevance.json") - + # Context relevance can only check against a single document at a time. document = docs[0] - + # First call triggers adapter loading result = rag.check_context_relevance(context, question, document, backend) - assert pytest.approx(result, 2e-2) == 0.45 + assert pytest.approx(result, abs=2e-2) == 0.45 # Second call hits a different code path from the first one result = rag.check_context_relevance(context, question, document, backend) - assert pytest.approx(result, 2e-2) == 0.45 - + assert pytest.approx(result, abs=2e-2) == 0.45 + + +def test_hallucination_detection(backend): + """Verify that the hallucination detection intrinsic functions properly.""" + context, assistant_response, docs = _read_input_json("hallucination_detection.json") + expected = _read_output_json("hallucination_detection.json") + + # First call triggers adapter loading + result = rag.flag_hallucinated_content(context, assistant_response, docs, backend) + # pytest.approx() chokes on lists of records, so we do this complicated dance. + for r, e in zip(result, expected, strict=True): + assert pytest.approx(r, abs=2e-2) == e + + # Second call hits a different code path from the first one + result = rag.flag_hallucinated_content(context, assistant_response, docs, backend) + for r, e in zip(result, expected, strict=True): + assert pytest.approx(r, abs=2e-2) == e + + +def test_answer_relevance(backend): + """Verify that the answer relevance composite intrinsic functions properly.""" + context, answer, docs = _read_input_json("answer_relevance.json") + expected_rewrite = "Alice, Bob, and Carol attended the meeting." + + # First call triggers adapter loading + result = rag.rewrite_answer_for_relevance(context, answer, docs, backend) + assert result == expected_rewrite + + # Second call hits a different code path from the first one + result = rag.rewrite_answer_for_relevance(context, answer, docs, backend) + assert result == expected_rewrite + + # Canned input always gets rewritten. Set threshold to disable the rewrite. + result = rag.rewrite_answer_for_relevance(context, answer, docs, backend, + rewrite_threshold=0.0) + assert result == answer if __name__ == "__main__": pytest.main([__file__]) diff --git a/test/stdlib_intrinsics/test_rag/testdata/input_json/answer_relevance.json b/test/stdlib_intrinsics/test_rag/testdata/input_json/answer_relevance.json new file mode 100644 index 000000000..30779c0c3 --- /dev/null +++ b/test/stdlib_intrinsics/test_rag/testdata/input_json/answer_relevance.json @@ -0,0 +1,13 @@ +{ + "messages": [ + {"role": "user", "content": "Who attended the meeting?"}, + {"role": "assistant", "content": "Many people attended the meeting."} + ], + "extra_body": { + "documents": [ + {"doc_id": "1", "text": "Meeting attendees: Alice, Bob, Carol."}, + {"doc_id": "2", "text": "Meeting time: 9:00 am to 11:00 am."} + ] + }, + "temperature": 0.0 +} \ No newline at end of file diff --git a/test/stdlib_intrinsics/test_rag/testdata/input_json/hallucination_detection.json b/test/stdlib_intrinsics/test_rag/testdata/input_json/hallucination_detection.json new file mode 100644 index 000000000..f224ed20a --- /dev/null +++ b/test/stdlib_intrinsics/test_rag/testdata/input_json/hallucination_detection.json @@ -0,0 +1,24 @@ +{ + "messages": [ + { + "role": "assistant", + "content": "Hello there, how can I help you?" + }, + { + "content": "Tell me about some yellow fish.", + "role": "user" + }, + { + "role": "assistant", + "content": "Purple bumble fish are yellow. Green bumble fish are also yellow." + } + ], + "extra_body": { + "documents": [ + { + "doc_id": "1", + "text": "The only type of fish that is yellow is the purple bumble fish." + } + ] + } +} \ No newline at end of file diff --git a/test/stdlib_intrinsics/test_rag/testdata/output_json/hallucination_detection.json b/test/stdlib_intrinsics/test_rag/testdata/output_json/hallucination_detection.json new file mode 100644 index 000000000..7546817e4 --- /dev/null +++ b/test/stdlib_intrinsics/test_rag/testdata/output_json/hallucination_detection.json @@ -0,0 +1,11 @@ +{ + "choices": [ + { + "index": 0, + "message": { + "content": "[{\"response_begin\": 0, \"response_end\": 36, \"response_text\": \"Purple bumble fish are yellow. \", \"faithfulness_likelihood\": 0.2062460112028628, \"explanation\": \"This sentence makes a factual claim about the color of fish. However, the provided document only mentions one type of fish that is yellow, which is the purple bumble fish. There is no information about green bumble fish in the document, so the claim about green bumble fish being yellow cannot be verified.\"}, {\"response_begin\": 36, \"response_end\": 70, \"response_text\": \"Green bumble fish are also yellow.\", \"faithfulness_likelihood\": 0.006380047389365753, \"explanation\": \"This sentence makes a factual claim about the color of fish. However, the provided document only mentions one type of fish that is yellow, which is the purple bumble fish. There is no information about green bumble fish in the document, so the claim about green bumble fish being yellow cannot be verified.\"}]", + "role": "assistant" + } + } + ] +} \ No newline at end of file From d9ac0852553a44a818cc0c6668a8a6c9fce8b39b Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Mon, 10 Nov 2025 17:53:37 -0800 Subject: [PATCH 19/38] Fix broken test after merge Signed-off-by: Fred Reiss --- test/backends/test_huggingface.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index d0947014d..b9190f815 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -116,7 +116,6 @@ def test_constraint_lora_override_does_not_override_alora(session, backend): val_result = validation_outputs[0] assert isinstance(val_result, ValidationResult) assert "requirement_likelihood" in str(val_result.reason) - assert str(val_result.reason) in ["Y", "N"] # Ensure the ValidationResult has its thunk and context set. Ensure the context has # the correct actions / results in it. @@ -196,7 +195,7 @@ class Email(pydantic.BaseModel): print(email) print("address:", email.to.email_address) - assert "@" in email.to.email_address, "The @ sign should be in the meail address." + assert "@" in email.to.email_address, "The @ sign should be in the email address." assert email.to.email_address.endswith("example.com"), ( "The email address should be at example.com" ) From da6de1e6c3db9638ba21303d9f3656b76307648d Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Tue, 11 Nov 2025 11:21:56 -0800 Subject: [PATCH 20/38] Make linter happy and allow GPU usage with HF backend Signed-off-by: Fred Reiss --- mellea/backends/huggingface.py | 2 +- mellea/backends/openai.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index 2cba5af1d..6f0b603b4 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -438,7 +438,7 @@ def _generate_from_context_standard( add_generation_prompt=True, # If we change this, must modify huggingface granite guardian. return_tensors="pt", **self._make_backend_specific_and_remove(model_options), - ) + ).to(self._model.device) format_kwargs = {} if _format: diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index eaebd1bee..77ba130b7 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -7,8 +7,8 @@ import inspect import json from collections.abc import Callable, Coroutine -from enum import Enum from copy import deepcopy +from enum import Enum from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlparse From 765c91f9310537d0f37e736aca432f91a36c3830 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Tue, 11 Nov 2025 11:40:18 -0800 Subject: [PATCH 21/38] Fix stuff that was missed during manual merge Signed-off-by: Fred Reiss --- mellea/backends/huggingface.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index 6f0b603b4..bc04c30b8 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -161,15 +161,23 @@ def __init__( self._hf_model_id = model_id.hf_model_name match custom_config: case None: + # Choose a device. + self._device = torch.device( + "cuda" + if torch.cuda.is_available() + else "mps" + if torch.backends.mps.is_available() + else "cpu" + ) # Get the model and tokenizer. self._model: PreTrainedModel = AutoModelForCausalLM.from_pretrained( self._hf_model_id - ) + ).to(self._device) # type: ignore self._tokenizer: PreTrainedTokenizer = AutoTokenizer.from_pretrained( self._hf_model_id ) case _: - self._tokenizer, self._model, _ = custom_config + self._tokenizer, self._model, self._device = custom_config self._use_caches = use_caches self._cache = cache if cache is not None else SimpleLRUCache(3) @@ -438,7 +446,7 @@ def _generate_from_context_standard( add_generation_prompt=True, # If we change this, must modify huggingface granite guardian. return_tensors="pt", **self._make_backend_specific_and_remove(model_options), - ).to(self._model.device) + ).to(self._device) # type: ignore format_kwargs = {} if _format: @@ -590,7 +598,7 @@ async def post_processing( cache_info = HFAloraCacheInfo( kv_cache=cache, merged_token_ids=output_complete, - merged_attention=torch.ones_like(output_complete), + merged_attention=torch.ones_like(output_complete).to(self._device), q_end=len(input_ids[0]), # type: ignore ) @@ -658,7 +666,9 @@ def generate_from_raw( prompts = [self.formatter.print(action) for action in actions] # batch-encoding call is deprecated in favor of this - inputs = self._tokenizer(prompts, return_tensors="pt", padding=True) + inputs = self._tokenizer(prompts, return_tensors="pt", padding=True).to( + self._device + ) if format is None: outputs = self._model.generate( # type: ignore From 0582f24426442549afa6ca076af4a596508a14ee Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Tue, 11 Nov 2025 11:46:34 -0800 Subject: [PATCH 22/38] More manual merge issues Signed-off-by: Fred Reiss --- mellea/backends/process_reward_models/huggingface/prms.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mellea/backends/process_reward_models/huggingface/prms.py b/mellea/backends/process_reward_models/huggingface/prms.py index c8c7be780..2525b8e6a 100644 --- a/mellea/backends/process_reward_models/huggingface/prms.py +++ b/mellea/backends/process_reward_models/huggingface/prms.py @@ -71,7 +71,7 @@ def score(self, query: str, response: str) -> tuple[list[float], list[list[float # move each item of the batch to the device for i in batches: - batches[i] = batches[i] + batches[i] = batches[i].to(self.model.device) with torch.no_grad(): model_outputs = self.model(**batches) @@ -178,7 +178,7 @@ def __init__( # initialize PRM head self.prm_head = torch.nn.Linear( self.model.config.hidden_size, 2, bias=False, dtype=self.model.dtype - ) + ).to(self.model.device) state = torch.load(model_name_or_path + "/added_params.bin") # need to do this-- we save model dict as `prm_head.weight` during training @@ -205,7 +205,7 @@ def score(self, query: str, response: str) -> tuple[list[float], list[list[float batch = self.prepare_inputs(query, list_of_steps) # move each item of the batch to the device for i in batch: - batch[i] = batch[i] + batch[i] = batch[i].to(self.model.device) with torch.no_grad(): model_outputs = self.model(**batch, output_hidden_states=True) From 869459eefde2a8ff136be1d53c9642df6d1dec60 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Tue, 11 Nov 2025 11:51:38 -0800 Subject: [PATCH 23/38] More manual merge issues Signed-off-by: Fred Reiss --- mellea/backends/huggingface.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index bc04c30b8..76a432cd9 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -182,7 +182,7 @@ def __init__( self._use_caches = use_caches self._cache = cache if cache is not None else SimpleLRUCache(3) - # Adapters can be made know to the backend (added) and loaded. + # Adapters can be made known to the backend (added) and loaded. self._added_adapters: dict[str, LocalHFAdapter] = {} self._loaded_adapters: dict[str, LocalHFAdapter] = {} @@ -913,6 +913,19 @@ def __init__( """ super().__init__(model_name_or_path) + # auto-device if not more specific + self._device = device + if device is None: + device_name: str = ( + "cuda" + if torch.cuda.is_available() + else "mps" + if torch.backends.mps.is_available() + else "cpu" + ) + assert device_name is not None + self._device = torch.device(device_name) # type: ignore + self.model: PreTrainedModel = AutoModelForCausalLM.from_pretrained( self.model_name_or_path, torch_dtype=torch.bfloat16 ) From d5732c3b5d6a305053790651949164ff498af1b6 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Tue, 11 Nov 2025 13:05:00 -0800 Subject: [PATCH 24/38] Fix broken test Signed-off-by: Fred Reiss --- test/backends/test_huggingface.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index b9190f815..44ccad8d5 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -9,14 +9,20 @@ from mellea.backends.formatter import TemplateFormatter from mellea.backends.huggingface import LocalHFBackend from mellea.backends.types import ModelOption -from mellea.stdlib.base import CBlock, ChatContext, Context, ModelOutputThunk, SimpleContext +from mellea.stdlib.base import ( + CBlock, + ChatContext, + Context, + ModelOutputThunk, + SimpleContext, +) from mellea.stdlib.requirement import ( ALoraRequirement, LLMaJRequirement, Requirement, ValidationResult, default_output_to_bool, - REQUIREMENT_REPO_ID + REQUIREMENT_REPO_ID, ) @@ -30,9 +36,11 @@ def backend(): ) # Assertion to assuage the wrath of the linter assert backend.model_id is not None - backend.add_adapter(GraniteCommonAdapter(REQUIREMENT_REPO_ID, - "requirement_check", - base_model_name=backend.model_id)) + backend.add_adapter( + GraniteCommonAdapter( + REQUIREMENT_REPO_ID, "requirement_check", base_model_name=backend.model_id + ) + ) return backend @@ -43,6 +51,7 @@ def session(backend): yield session session.reset() + def test_adapters(backend): assert len(backend._added_adapters.items()) > 0 @@ -59,6 +68,7 @@ def test_adapters(backend): backend.unload_adapter(adapter.qualified_name) assert adapter.qualified_name not in backend._loaded_adapters + @pytest.mark.qualitative def test_system_prompt(session): result = session.chat( @@ -184,7 +194,10 @@ class Email(pydantic.BaseModel): body: str output = session.instruct( - "Write a short email to Olivia, thanking her for organizing a sailing activity. Her email server is example.com. No more than two sentences. ", + "Write a short email to Olivia, thanking her for organizing a sailing " + "activity. " + "Her email is olivia@example.com. " + "No more than two sentences. ", format=Email, model_options={ModelOption.MAX_NEW_TOKENS: 2**8}, ) @@ -200,6 +213,7 @@ class Email(pydantic.BaseModel): "The email address should be at example.com" ) + @pytest.mark.qualitative def test_generate_from_raw(session): prompts = [ @@ -216,6 +230,7 @@ def test_generate_from_raw(session): assert len(results) == len(prompts) + @pytest.mark.qualitative def test_generate_from_raw_with_format(session): prompts = ["what is 1+1?", "what is 2+2?", "what is 3+3?", "what is 4+4?"] From ac9ee44e4d4101946c08699d9aeeb1681d4c70fa Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Tue, 11 Nov 2025 13:17:33 -0800 Subject: [PATCH 25/38] Adjust type hints Signed-off-by: Fred Reiss --- mellea/stdlib/intrinsics/rag.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/mellea/stdlib/intrinsics/rag.py b/mellea/stdlib/intrinsics/rag.py index 7fa412733..448464d76 100644 --- a/mellea/stdlib/intrinsics/rag.py +++ b/mellea/stdlib/intrinsics/rag.py @@ -5,7 +5,7 @@ import granite_common -import mellea.stdlib.funcs +import mellea.stdlib.funcs as mfuncs from mellea.backends import Backend from mellea.backends.adapters.adapter import ( AdapterMixin, @@ -44,7 +44,11 @@ def _call_intrinsic( - intrinsic_name: str, context: ChatContext, backend, /, kwargs: dict | None = None + intrinsic_name: str, + context: ChatContext, + backend: AdapterMixin, # type: ignore + /, + kwargs: dict | None = None, ): """Shared code for invoking intrinsics. @@ -59,7 +63,7 @@ def _call_intrinsic( f"Should be one of {list(_CATALOG.keys())}" ) repo_id, adapter_type = _CATALOG[intrinsic_name] - base_model_name = backend.model_id + base_model_name = backend.model_id # type: ignore if base_model_name is None: raise ValueError("Backend has no model ID") adapter = GraniteCommonAdapter( @@ -74,10 +78,10 @@ def _call_intrinsic( ) # Execute the AST node. - model_output_thunk, _ = mellea.stdlib.funcs.act( + model_output_thunk, _ = mfuncs.act( intrinsic, context, - backend, + backend, # type: ignore # No rejection sampling, please strategy=None, ) @@ -98,7 +102,7 @@ def check_answerability( context: ChatContext, question: str, documents: collections.abc.Iterable[Document], - backend, # Can't put type hints here because linter complains + backend: AdapterMixin, ) -> float: """Test a user's question for answerability. @@ -124,9 +128,7 @@ def check_answerability( def rewrite_question( - context: ChatContext, - question: str, - backend, # Can't put type hints here because linter complains + context: ChatContext, question: str, backend: AdapterMixin ) -> float: """Rewrite a user's question for retrieval. @@ -150,7 +152,7 @@ def find_citations( context: ChatContext, response: str, documents: collections.abc.Iterable[Document], - backend, # Can't put type hints here because linter complains + backend: AdapterMixin, ) -> list[dict]: """Find information in documents that supports an assistant response. @@ -184,10 +186,7 @@ def find_citations( def check_context_relevance( - context: ChatContext, - question: str, - document: Document, - backend, # Can't put type hints here because linter complains + context: ChatContext, question: str, document: Document, backend: AdapterMixin ) -> float: """Test whether a document is relevant to a user's question. @@ -217,7 +216,7 @@ def flag_hallucinated_content( context: ChatContext, response: str, documents: collections.abc.Iterable[Document], - backend, # Can't put type hints here because linter complains + backend: AdapterMixin, ) -> float: """Flag potentially-hallucinated sentences in an agent's response. @@ -251,7 +250,7 @@ def rewrite_answer_for_relevance( context, response: str, documents: collections.abc.Iterable[Document], - backend, # Can't put type hints here because linter complains + backend: AdapterMixin, /, rewrite_threshold: float = 0.5, ) -> str: From 6c29eaea26e8f25afb1e3ea4379ff1fdb89e8609 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Tue, 11 Nov 2025 14:16:23 -0800 Subject: [PATCH 26/38] Add qualitative mark to tests Signed-off-by: Fred Reiss --- test/stdlib_intrinsics/test_rag/test_rag.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/stdlib_intrinsics/test_rag/test_rag.py b/test/stdlib_intrinsics/test_rag/test_rag.py index dc5c21ebb..246ec3739 100644 --- a/test/stdlib_intrinsics/test_rag/test_rag.py +++ b/test/stdlib_intrinsics/test_rag/test_rag.py @@ -80,6 +80,7 @@ def _read_output_json(file_name: str): return json.loads(result_str) +@pytest.mark.qualitative def test_answerability(backend): """Verify that the answerability intrinsic functions properly.""" context, next_user_turn, documents = _read_input_json("answerability.json") @@ -93,6 +94,7 @@ def test_answerability(backend): assert pytest.approx(result) == 1.0 +@pytest.mark.qualitative def test_query_rewrite(backend): """Verify that the answerability intrinsic functions properly.""" context, next_user_turn, _ = _read_input_json("query_rewrite.json") @@ -110,6 +112,7 @@ def test_query_rewrite(backend): assert result == expected +@pytest.mark.qualitative def test_citations(backend): """Verify that the citations intrinsic functions properly.""" context, assistant_response, docs = _read_input_json("citations.json") @@ -124,6 +127,7 @@ def test_citations(backend): assert result == expected +@pytest.mark.qualitative def test_context_relevance(backend): """Verify that the context relevance intrinsic functions properly.""" context, question, docs = _read_input_json("context_relevance.json") @@ -140,6 +144,7 @@ def test_context_relevance(backend): assert pytest.approx(result, abs=2e-2) == 0.45 +@pytest.mark.qualitative def test_hallucination_detection(backend): """Verify that the hallucination detection intrinsic functions properly.""" context, assistant_response, docs = _read_input_json("hallucination_detection.json") @@ -157,6 +162,7 @@ def test_hallucination_detection(backend): assert pytest.approx(r, abs=2e-2) == e +@pytest.mark.qualitative def test_answer_relevance(backend): """Verify that the answer relevance composite intrinsic functions properly.""" context, answer, docs = _read_input_json("answer_relevance.json") From 9ce2e3237994d2b7482875a7721880a3c643c3eb Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Tue, 11 Nov 2025 17:19:04 -0800 Subject: [PATCH 27/38] Change import of obtain_lora() Signed-off-by: Fred Reiss --- mellea/backends/adapters/adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 52002bb43..dcafcb3b9 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -130,7 +130,7 @@ def __init__( ) if config_file is None and config_dict is None: is_alora = self.adapter_type == AdapterType.ALORA - config_file = granite_common.intrinsics.util.obtain_io_yaml( + config_file = granite_common.intrinsics.obtain_io_yaml( self.intrinsic_name, self.base_model_name, alora=is_alora, @@ -195,7 +195,7 @@ def download_and_get_path(self, base_model_name: str) -> str: """ is_alora = self.adapter_type == AdapterType.ALORA return str( - granite_common.intrinsics.util.obtain_lora( + granite_common.intrinsics.obtain_lora( self.intrinsic_name, base_model_name, alora=is_alora, From c33d4400dc55ea27799ff6acd2b8c53e28aaf0b0 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Tue, 11 Nov 2025 17:21:54 -0800 Subject: [PATCH 28/38] Update import Signed-off-by: Fred Reiss --- mellea/backends/adapters/adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index dcafcb3b9..f5e854947 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -5,7 +5,7 @@ from enum import Enum from typing import Any, TypeVar -import granite_common +import granite_common.intrinsics import yaml from litellm import cast From 2c37d620fc991c28e004cc4f99c6133dc3585d40 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Wed, 12 Nov 2025 13:43:51 -0800 Subject: [PATCH 29/38] Update code after breaking API changes Signed-off-by: Fred Reiss --- docs/examples/intrinsics/intrinsics.py | 2 +- mellea/stdlib/intrinsics/rag.py | 2 +- mellea/stdlib/sampling/budget_forcing.py | 2 +- uv.lock | 100 +++++++++++------------ 4 files changed, 53 insertions(+), 53 deletions(-) diff --git a/docs/examples/intrinsics/intrinsics.py b/docs/examples/intrinsics/intrinsics.py index 98fd46af7..ad572ba2b 100644 --- a/docs/examples/intrinsics/intrinsics.py +++ b/docs/examples/intrinsics/intrinsics.py @@ -2,7 +2,7 @@ from mellea.backends.adapters.adapter import AdapterType, GraniteCommonAdapter from mellea.stdlib.base import ChatContext, ModelOutputThunk from mellea.stdlib.chat import Message -import mellea.stdlib.funcs as mfuncs +import mellea.stdlib.functional as mfuncs from mellea.stdlib.intrinsics.intrinsic import Intrinsic # Create the Adapter. GraniteCommonAdapter's default to ALORAs. diff --git a/mellea/stdlib/intrinsics/rag.py b/mellea/stdlib/intrinsics/rag.py index 448464d76..06817a6da 100644 --- a/mellea/stdlib/intrinsics/rag.py +++ b/mellea/stdlib/intrinsics/rag.py @@ -5,7 +5,7 @@ import granite_common -import mellea.stdlib.funcs as mfuncs +import mellea.stdlib.functional as mfuncs from mellea.backends import Backend from mellea.backends.adapters.adapter import ( AdapterMixin, diff --git a/mellea/stdlib/sampling/budget_forcing.py b/mellea/stdlib/sampling/budget_forcing.py index a1be15a2a..21dbc504e 100644 --- a/mellea/stdlib/sampling/budget_forcing.py +++ b/mellea/stdlib/sampling/budget_forcing.py @@ -4,10 +4,10 @@ import tqdm +import mellea.stdlib.functional as mfuncs from mellea.backends import Backend, BaseModelSubclass from mellea.backends.ollama import OllamaModelBackend from mellea.helpers.fancy_logger import FancyLogger -from mellea.stdlib import funcs as mfuncs from mellea.stdlib.base import ModelOutputThunk from mellea.stdlib.requirement import Requirement, ValidationResult from mellea.stdlib.sampling import RejectionSamplingStrategy, SamplingResult diff --git a/uv.lock b/uv.lock index 636e096fd..326315ab5 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and python_full_version < '4' and sys_platform == 'darwin'", @@ -850,7 +850,7 @@ name = "colorlog" version = "6.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or (python_full_version >= '4' and sys_platform == 'win32')" }, + { name = "colorama", marker = "python_full_version < '3.14' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/61/f083b5ac52e505dfc1c624eafbf8c7589a0d7f32daa398d2e7590efa5fda/colorlog-6.10.1.tar.gz", hash = "sha256:eb4ae5cb65fe7fec7773c2306061a8e63e02efc2c72eba9d27b0fa23c94f1321", size = 17162, upload-time = "2025-10-16T16:14:11.978Z" } wheels = [ @@ -1727,28 +1727,28 @@ wheels = [ ] [[package]] -name = "granite-common" -version = "0.3.2" +name = "googleapis-common-protos" +version = "1.72.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jsonschema" }, - { name = "pydantic" }, + { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/70/9b/c0e846c0517c7581e63901f90f23a28aa6758ec2b686171a5f23ba73ae48/granite_common-0.3.2.tar.gz", hash = "sha256:f5bc850573700f160bab0ae921d5156a5fd4594ed6806ae82ddf8a6043aa4331", size = 273140, upload-time = "2025-10-24T19:29:51.449Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/7b/adfd75544c415c487b33061fe7ae526165241c1ea133f9a9125a56b39fd8/googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5", size = 147433, upload-time = "2025-11-06T18:29:24.087Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/54/cf965d50fe493f4fb8ab1d4dea371a5d9d74d8e0bfdf2bd17f41a3af973b/granite_common-0.3.2-py3-none-any.whl", hash = "sha256:45d5a99e264f9e009215daf039c85a1e1ea216983962bcff2c75b4fa4a815d87", size = 77490, upload-time = "2025-10-24T19:29:49.97Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ab/09169d5a4612a5f92490806649ac8d41e3ec9129c636754575b3553f4ea4/googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038", size = 297515, upload-time = "2025-11-06T18:29:13.14Z" }, ] [[package]] -name = "googleapis-common-protos" -version = "1.72.0" +name = "granite-common" +version = "0.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "protobuf" }, + { name = "jsonschema" }, + { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e5/7b/adfd75544c415c487b33061fe7ae526165241c1ea133f9a9125a56b39fd8/googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5", size = 147433, upload-time = "2025-11-06T18:29:24.087Z" } +sdist = { url = "https://files.pythonhosted.org/packages/39/7d/d03e3c55651476cb2838c6051457063d4f5fc448700054b73ef2edc28624/granite_common-0.3.4.tar.gz", hash = "sha256:5d362f1bd3e818ce2f6c280a51c730493b5a15b45d3d5bfdc4221bc856cf2b25", size = 273862, upload-time = "2025-11-12T01:17:15.285Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/ab/09169d5a4612a5f92490806649ac8d41e3ec9129c636754575b3553f4ea4/googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038", size = 297515, upload-time = "2025-11-06T18:29:13.14Z" }, + { url = "https://files.pythonhosted.org/packages/4f/18/715fa6eb7fabb06c7040b65e6ff8eb2bb787e266c75f1af9acbe5e3a1260/granite_common-0.3.4-py3-none-any.whl", hash = "sha256:85fdbf8b96fb667d6612959af863dc3cfd3a6c0744d9e3ad9aa52aee3e4a2909", size = 77557, upload-time = "2025-11-12T01:17:13.708Z" }, ] [[package]] @@ -3327,10 +3327,10 @@ requires-dist = [ { name = "numpy", marker = "extra == 'vllm'", specifier = "<2.0.0" }, { name = "ollama", specifier = ">=0.5.1" }, { name = "openai" }, - { name = "peft", marker = "extra == 'hf'", git = "https://github.com/huggingface/peft.git?rev=293aea5df6db240856a77f89955d1a89ce38b50d" }, { name = "outlines", marker = "extra == 'hf'" }, { name = "outlines-core", marker = "extra == 'hf'", specifier = "==0.1.26" }, { name = "outlines-core", marker = "extra == 'vllm'", specifier = "==0.1.26" }, + { name = "peft", marker = "extra == 'hf'", git = "https://github.com/huggingface/peft.git?rev=293aea5df6db240856a77f89955d1a89ce38b50d" }, { name = "pillow" }, { name = "pydantic" }, { name = "requests", specifier = ">=2.32.3" }, @@ -4081,7 +4081,7 @@ name = "nvidia-cudnn-cu12" version = "9.5.1.17" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/2a/78/4535c9c7f859a64781e43c969a3a7e84c54634e319a996d43ef32ce46f83/nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2", size = 570988386, upload-time = "2024-10-25T19:54:26.39Z" }, @@ -4092,7 +4092,7 @@ name = "nvidia-cufft-cu12" version = "11.3.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8f/16/73727675941ab8e6ffd86ca3a4b7b47065edcca7a997920b831f8147c99d/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ccba62eb9cef5559abd5e0d54ceed2d9934030f51163df018532142a8ec533e5", size = 200221632, upload-time = "2024-11-20T17:41:32.357Z" }, @@ -4121,9 +4121,9 @@ name = "nvidia-cusolver-cu12" version = "11.7.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, - { name = "nvidia-cusparse-cu12" }, - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-cusparse-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f0/6e/c2cf12c9ff8b872e92b4a5740701e51ff17689c4d726fca91875b07f655d/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c", size = 158229790, upload-time = "2024-11-20T17:43:43.211Z" }, @@ -4135,7 +4135,7 @@ name = "nvidia-cusparse-cu12" version = "12.5.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/06/1e/b8b7c2f4099a37b96af5c9bb158632ea9e5d9d27d7391d7eb8fc45236674/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7556d9eca156e18184b94947ade0fba5bb47d69cec46bf8660fd2c71a4b48b73", size = 216561367, upload-time = "2024-11-20T17:44:54.824Z" }, @@ -4180,9 +4180,9 @@ name = "ocrmac" version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "pillow" }, - { name = "pyobjc-framework-vision" }, + { name = "click", marker = "sys_platform == 'darwin'" }, + { name = "pillow", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/dd/dc/de3e9635774b97d9766f6815bbb3f5ec9bce347115f10d9abbf2733a9316/ocrmac-1.0.0.tar.gz", hash = "sha256:5b299e9030c973d1f60f82db000d6c2e5ff271601878c7db0885e850597d1d2e", size = 1463997, upload-time = "2024-11-07T12:00:00.197Z" } wheels = [ @@ -4207,8 +4207,8 @@ name = "omegaconf" version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "antlr4-python3-runtime", version = "4.9.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "pyyaml", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "antlr4-python3-runtime", version = "4.9.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, + { name = "pyyaml", marker = "python_full_version < '3.14'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/09/48/6388f1bb9da707110532cb70ec4d2822858ddfb44f1cdf1233c20a80ea4b/omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7", size = 3298120, upload-time = "2022-12-08T20:59:22.753Z" } wheels = [ @@ -4239,7 +4239,7 @@ name = "opencv-python" version = "4.11.0.86" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "numpy", marker = "python_full_version < '3.14'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/17/06/68c27a523103dad5837dc5b87e71285280c4f098c60e4fe8a8db6486ab09/opencv-python-4.11.0.86.tar.gz", hash = "sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4", size = 95171956, upload-time = "2025-01-16T13:52:24.737Z" } wheels = [ @@ -5339,7 +5339,7 @@ name = "pyobjc-framework-cocoa" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/37/6f/89837da349fe7de6476c426f118096b147de923139556d98af1832c64b97/pyobjc_framework_cocoa-12.0.tar.gz", hash = "sha256:02d69305b698015a20fcc8e1296e1528e413d8cf9fdcd590478d359386d76e8a", size = 2771906, upload-time = "2025-10-21T08:30:51.765Z" } wheels = [ @@ -5357,8 +5357,8 @@ name = "pyobjc-framework-coreml" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0c/a0/875b5174794c984df60944be54df0282945f8bae4a606fbafa0c6b717ddd/pyobjc_framework_coreml-12.0.tar.gz", hash = "sha256:e1d7a9812886150881c86000fba885cb15201352c75fb286bd9e3a1819b5a4d5", size = 40814, upload-time = "2025-10-21T08:31:53.83Z" } wheels = [ @@ -5376,8 +5376,8 @@ name = "pyobjc-framework-quartz" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/91/0b/3c34fc9de790daff5ca49d1f36cb8dcc353ac10e4e29b4759e397a3831f4/pyobjc_framework_quartz-12.0.tar.gz", hash = "sha256:5bcb9e78d671447e04d89e2e3c39f3135157892243facc5f8468aa333e40d67f", size = 3159509, upload-time = "2025-10-21T08:40:01.918Z" } wheels = [ @@ -5395,10 +5395,10 @@ name = "pyobjc-framework-vision" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, - { name = "pyobjc-framework-coreml" }, - { name = "pyobjc-framework-quartz" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-coreml", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/5a/07cdead5adb77d0742b014fa742d503706754e3ad10e39760e67bb58b497/pyobjc_framework_vision-12.0.tar.gz", hash = "sha256:942c9583f1d887ac9f704f3b0c21b3206b68e02852a87219db4309bb13a02f14", size = 59905, upload-time = "2025-10-21T08:41:53.741Z" } wheels = [ @@ -5856,17 +5856,17 @@ name = "rapidocr" version = "3.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorlog", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "numpy", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "omegaconf", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "opencv-python", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "pillow", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "pyclipper", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "pyyaml", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "requests", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "shapely", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "six", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "tqdm", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "colorlog", marker = "python_full_version < '3.14'" }, + { name = "numpy", marker = "python_full_version < '3.14'" }, + { name = "omegaconf", marker = "python_full_version < '3.14'" }, + { name = "opencv-python", marker = "python_full_version < '3.14'" }, + { name = "pillow", marker = "python_full_version < '3.14'" }, + { name = "pyclipper", marker = "python_full_version < '3.14'" }, + { name = "pyyaml", marker = "python_full_version < '3.14'" }, + { name = "requests", marker = "python_full_version < '3.14'" }, + { name = "shapely", marker = "python_full_version < '3.14'" }, + { name = "six", marker = "python_full_version < '3.14'" }, + { name = "tqdm", marker = "python_full_version < '3.14'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/3c/83/5b8c8075954c5b61d938b8954710d986134c4ca7c32a841ad7d8c844cf6c/rapidocr-3.4.2-py3-none-any.whl", hash = "sha256:17845fa8cc9a20a935111e59482f2214598bba1547000cfd960d8924dd4522a5", size = 15056674, upload-time = "2025-10-11T14:43:00.296Z" }, @@ -6806,7 +6806,7 @@ name = "shapely" version = "2.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "python_full_version < '4'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/bc/0989043118a27cccb4e906a46b7565ce36ca7b57f5a18b78f4f1b0f72d9d/shapely-2.1.2.tar.gz", hash = "sha256:2ed4ecb28320a433db18a5bf029986aa8afcfd740745e78847e330d5d94922a9", size = 315489, upload-time = "2025-09-24T13:51:41.432Z" } wheels = [ @@ -7563,7 +7563,7 @@ name = "triton" version = "3.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "setuptools" }, + { name = "setuptools", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/76/04/d54d3a6d077c646624dc9461b0059e23fd5d30e0dbe67471e3654aec81f9/triton-3.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fad99beafc860501d7fcc1fb7045d9496cbe2c882b1674640304949165a916e7", size = 156441993, upload-time = "2025-04-09T20:27:25.107Z" }, @@ -8089,8 +8089,8 @@ name = "xformers" version = "0.0.30" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, - { name = "torch" }, + { name = "numpy", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "torch", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bf/f7/dd2269cce89fd1221947dd7cc3a60707ffe721ef55c1803ac3b1a1f7ae5c/xformers-0.0.30.tar.gz", hash = "sha256:a12bf3eb39e294cdbe8a7253ac9b665f41bac61d6d98df174e34ef7bdb6f2fc4", size = 10214139, upload-time = "2025-04-28T20:51:02.045Z" } wheels = [ From 0bb208749819814c1df2973fb4cb63808acef296 Mon Sep 17 00:00:00 2001 From: Jake LoRocco Date: Thu, 13 Nov 2025 08:21:11 -0500 Subject: [PATCH 30/38] feat: small changes to intrinsics; along with fixes to docs / tests --- docs/dev/requirement_aLoRA_rerouting.md | 2 +- docs/examples/intrinsics/intrinsics.py | 11 +- mellea/backends/adapters/adapter.py | 16 ++- mellea/backends/huggingface.py | 18 +++- mellea/backends/openai.py | 8 +- mellea/stdlib/intrinsics/rag.py | 20 ++-- mellea/stdlib/sampling/budget_forcing.py | 2 +- test/backends/test_huggingface.py | 4 +- .../test_openai_vllm/test_openai_vllm.py | 10 +- test/stdlib_intrinsics/test_rag/test_rag.py | 26 ++--- uv.lock | 100 +++++++++--------- 11 files changed, 122 insertions(+), 95 deletions(-) diff --git a/docs/dev/requirement_aLoRA_rerouting.md b/docs/dev/requirement_aLoRA_rerouting.md index d21fb7770..011073bdc 100644 --- a/docs/dev/requirement_aLoRA_rerouting.md +++ b/docs/dev/requirement_aLoRA_rerouting.md @@ -39,7 +39,7 @@ m = start_session( "huggingface.LocalHFBackend:ibm-granite/granite-3.2-8b-instruct") # By default, the AloraRequirement uses a GraniteCommonAdapter with "requirement_check". -m.backend.add_adapter(GraniteCommonAdapter("requirement_check")) +m.backend.add_adapter(GraniteCommonAdapter("ibm-granite/rag-intrinsics-lib", "requirement_check", base_model_name="granite-3.2-8b-instruct")) m.instruct( "Corporate wants you to find the difference between these two strings:\n\naaa\naba") diff --git a/docs/examples/intrinsics/intrinsics.py b/docs/examples/intrinsics/intrinsics.py index ad572ba2b..32c256aea 100644 --- a/docs/examples/intrinsics/intrinsics.py +++ b/docs/examples/intrinsics/intrinsics.py @@ -2,11 +2,9 @@ from mellea.backends.adapters.adapter import AdapterType, GraniteCommonAdapter from mellea.stdlib.base import ChatContext, ModelOutputThunk from mellea.stdlib.chat import Message -import mellea.stdlib.functional as mfuncs +import mellea.stdlib.funcs as mfuncs from mellea.stdlib.intrinsics.intrinsic import Intrinsic - -# Create the Adapter. GraniteCommonAdapter's default to ALORAs. -req_adapter = GraniteCommonAdapter("requirement_check") +from mellea.stdlib.requirement import REQUIREMENT_REPO_ID # Create the backend. Assumes a locally running VLLM server. backend = OpenAIBackend( @@ -20,6 +18,11 @@ # adapters on the server. backend._server_type = _ServerType.REMOTE_VLLM +# Create the Adapter. GraniteCommonAdapter's default to ALORAs. +req_adapter = GraniteCommonAdapter( + REQUIREMENT_REPO_ID, "requirement_check", base_model_name=backend.base_model_name +) + # Add the adapter to the backend. backend.add_adapter(req_adapter) diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index f5e854947..5da86715b 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -123,12 +123,15 @@ def __init__( f"parameters provided. Values were {config_file=} " f"and {config_dict=}" ) - if config_file is None and config_dict is None and base_model_name is None: + if config_file is None and config_dict is None and self.base_model_name is None: raise ValueError( "At least one of [config_file, config_dict, base_model_name] " "must be provided." ) if config_file is None and config_dict is None: + assert self.base_model_name is not None, ( + "must provide `base_model_name` if not providing a `config_file` or `config_dict`" + ) is_alora = self.adapter_type == AdapterType.ALORA config_file = granite_common.intrinsics.obtain_io_yaml( self.intrinsic_name, @@ -240,18 +243,27 @@ def get_adapter_for_intrinsic( return adapter -class AdapterMixin(abc.ABC): +class AdapterMixin(Backend, abc.ABC): """Mixin class for backends capable of utilizing adapters.""" + @property + @abc.abstractmethod + def base_model_name(self) -> str: + """Returns the base_model_id of the model used by the backend. For example, `granite-3.3-8b-instruct` for `ibm-granite/granite-3.3-8b-instruct`.""" + + @abc.abstractmethod def add_adapter(self, *args, **kwargs): """Adds the given adapter to the backend. Must not have been added to a different backend.""" + @abc.abstractmethod def load_adapter(self, adapter_qualified_name: str): """Loads the given adapter for the backend. Must have previously been added.""" + @abc.abstractmethod def unload_adapter(self, adapter_qualified_name: str): """Unloads the given adapter from the backend.""" + @abc.abstractmethod def list_adapters(self) -> list[str]: """Lists the adapters added via add_adapter(). diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index 76a432cd9..39b7f1b36 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -828,6 +828,11 @@ def _filter_chat_template_only_options( return {k: v for k, v in model_options.items() if k not in chat_template_only} # region Adapter loading, unloading, and utility functions. + @property + def base_model_name(self): + """Returns the base_model_id of the model used by the backend. For example, `granite-3.3-8b-instruct` for `ibm-granite/granite-3.3-8b-instruct`.""" + return self._hf_model_id.split("/")[1] + def add_adapter(self, adapter: LocalHFAdapter): """Adds the given adapter to the backend. Must not have been added to a different backend.""" if adapter.backend is not None: @@ -847,8 +852,7 @@ def add_adapter(self, adapter: LocalHFAdapter): ) return None - base_model_name = self._hf_model_id.split("/")[1] - adapter.path = adapter.get_local_hf_path(base_model_name) + adapter.path = adapter.get_local_hf_path(self.base_model_name) adapter.backend = self self._added_adapters[adapter.qualified_name] = adapter @@ -861,7 +865,15 @@ def load_adapter(self, adapter_qualified_name: str): ) try: - self._model.load_adapter(adapter.path, adapter.qualified_name) + adapter_kwargs = {} + + # Peft tries to stringify the device. If it's mps, it gets stringified as "mps:0" which causes + # an error when loading with safetensors.torch.load_file. Force the device as a string "mps" to fix. + if self._device == torch.device("mps"): + adapter_kwargs["device"] = "mps" + self._model.load_adapter( + adapter.path, adapter.qualified_name, adapter_kwargs=adapter_kwargs + ) except ValueError as e: # If it's just that it's already loaded, ignore it. if f"Adapter with name {adapter_qualified_name} already exists." not in str( diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index 77ba130b7..da3a2fd84 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -860,6 +860,11 @@ def generate_from_raw( return results + @property + def base_model_name(self): + """Returns the base_model_id of the model used by the backend. For example, `granite-3.3-8b-instruct` for `ibm-granite/granite-3.3-8b-instruct`.""" + return self._hf_model_id.split("/")[1] + def add_adapter(self, adapter: OpenAIAdapter): """Adds the given adapter to the backend. Must not have been added to a different backend.""" if adapter.backend is not None: @@ -879,9 +884,8 @@ def add_adapter(self, adapter: OpenAIAdapter): ) return None - base_model_name = self._hf_model_id.split("/")[-1] adapter.path = adapter.get_open_ai_path( - base_model_name, server_type=self._server_type + self.base_model_name, server_type=self._server_type ) adapter.backend = self self._added_adapters[adapter.qualified_name] = adapter diff --git a/mellea/stdlib/intrinsics/rag.py b/mellea/stdlib/intrinsics/rag.py index 06817a6da..dadeed47f 100644 --- a/mellea/stdlib/intrinsics/rag.py +++ b/mellea/stdlib/intrinsics/rag.py @@ -5,7 +5,7 @@ import granite_common -import mellea.stdlib.functional as mfuncs +import mellea.stdlib.funcs as mfuncs from mellea.backends import Backend from mellea.backends.adapters.adapter import ( AdapterMixin, @@ -46,7 +46,7 @@ def _call_intrinsic( intrinsic_name: str, context: ChatContext, - backend: AdapterMixin, # type: ignore + backend: AdapterMixin, /, kwargs: dict | None = None, ): @@ -63,7 +63,7 @@ def _call_intrinsic( f"Should be one of {list(_CATALOG.keys())}" ) repo_id, adapter_type = _CATALOG[intrinsic_name] - base_model_name = backend.model_id # type: ignore + base_model_name = backend.base_model_name if base_model_name is None: raise ValueError("Backend has no model ID") adapter = GraniteCommonAdapter( @@ -81,7 +81,7 @@ def _call_intrinsic( model_output_thunk, _ = mfuncs.act( intrinsic, context, - backend, # type: ignore + backend, # No rejection sampling, please strategy=None, ) @@ -99,9 +99,9 @@ def _call_intrinsic( def check_answerability( - context: ChatContext, question: str, documents: collections.abc.Iterable[Document], + context: ChatContext, backend: AdapterMixin, ) -> float: """Test a user's question for answerability. @@ -128,7 +128,7 @@ def check_answerability( def rewrite_question( - context: ChatContext, question: str, backend: AdapterMixin + question: str, context: ChatContext, backend: AdapterMixin ) -> float: """Rewrite a user's question for retrieval. @@ -149,9 +149,9 @@ def rewrite_question( def find_citations( - context: ChatContext, response: str, documents: collections.abc.Iterable[Document], + context: ChatContext, backend: AdapterMixin, ) -> list[dict]: """Find information in documents that supports an assistant response. @@ -186,7 +186,7 @@ def find_citations( def check_context_relevance( - context: ChatContext, question: str, document: Document, backend: AdapterMixin + question: str, document: Document, context: ChatContext, backend: AdapterMixin ) -> float: """Test whether a document is relevant to a user's question. @@ -213,9 +213,9 @@ def check_context_relevance( def flag_hallucinated_content( - context: ChatContext, response: str, documents: collections.abc.Iterable[Document], + context: ChatContext, backend: AdapterMixin, ) -> float: """Flag potentially-hallucinated sentences in an agent's response. @@ -247,9 +247,9 @@ def flag_hallucinated_content( def rewrite_answer_for_relevance( - context, response: str, documents: collections.abc.Iterable[Document], + context: ChatContext, backend: AdapterMixin, /, rewrite_threshold: float = 0.5, diff --git a/mellea/stdlib/sampling/budget_forcing.py b/mellea/stdlib/sampling/budget_forcing.py index 21dbc504e..a1be15a2a 100644 --- a/mellea/stdlib/sampling/budget_forcing.py +++ b/mellea/stdlib/sampling/budget_forcing.py @@ -4,10 +4,10 @@ import tqdm -import mellea.stdlib.functional as mfuncs from mellea.backends import Backend, BaseModelSubclass from mellea.backends.ollama import OllamaModelBackend from mellea.helpers.fancy_logger import FancyLogger +from mellea.stdlib import funcs as mfuncs from mellea.stdlib.base import ModelOutputThunk from mellea.stdlib.requirement import Requirement, ValidationResult from mellea.stdlib.sampling import RejectionSamplingStrategy, SamplingResult diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index 44ccad8d5..6573e1ae4 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -34,11 +34,9 @@ def backend(): formatter=TemplateFormatter(model_id="ibm-granite/granite-4.0-tiny-preview"), cache=SimpleLRUCache(5), ) - # Assertion to assuage the wrath of the linter - assert backend.model_id is not None backend.add_adapter( GraniteCommonAdapter( - REQUIREMENT_REPO_ID, "requirement_check", base_model_name=backend.model_id + REQUIREMENT_REPO_ID, "requirement_check", base_model_name=backend.base_model_name ) ) return backend diff --git a/test/backends/test_openai_vllm/test_openai_vllm.py b/test/backends/test_openai_vllm/test_openai_vllm.py index f8fd86700..099bf0846 100644 --- a/test/backends/test_openai_vllm/test_openai_vllm.py +++ b/test/backends/test_openai_vllm/test_openai_vllm.py @@ -1,10 +1,9 @@ # test/rits_backend_tests/test_openai_integration.py -from contextvars import Context from mellea import MelleaSession from mellea.backends.adapters.adapter import GraniteCommonAdapter -from mellea.stdlib.base import CBlock, ModelOutputThunk, ChatContext +from mellea.stdlib.base import CBlock, ModelOutputThunk, ChatContext, Context from mellea.backends.openai import OpenAIBackend -from mellea.stdlib.requirement import Requirement, ALoraRequirement, LLMaJRequirement, req +from mellea.stdlib.requirement import REQUIREMENT_REPO_ID, Requirement, ALoraRequirement, LLMaJRequirement, req from mellea.backends.formatter import TemplateFormatter from mellea.backends.types import _ServerType, ModelOption @@ -136,14 +135,14 @@ class TestOpenAIALoraStuff: base_url="http://localhost:8000/v1", api_key="EMPTY", ) - backend.add_adapter(GraniteCommonAdapter("requirement_check")) + backend.add_adapter(GraniteCommonAdapter(REQUIREMENT_REPO_ID, "requirement_check", base_model_name=backend.base_model_name)) m = MelleaSession(backend, ctx=ChatContext()) def test_adapters(self): assert len(self.backend._added_adapters.items()) > 0 - adapter = self.backend._added_adapters["requirement_check_alora"] + adapter = self.backend._added_adapters[f"{REQUIREMENT_REPO_ID}_requirement_check_alora"] self.backend.load_adapter(adapter.qualified_name) assert adapter.qualified_name in self.backend._loaded_adapters @@ -209,7 +208,6 @@ def test_constraint_lora_override_does_not_override_alora(self): assert len(validation_outputs) == 1 non_alora_output = validation_outputs[0] assert "requirement_likelihood" in str(non_alora_output.reason) - assert str(non_alora_output.reason) in ["Y", "N"] # Ensure the ValidationResult has its thunk and context set. Ensure the context has # the correct actions / results in it. diff --git a/test/stdlib_intrinsics/test_rag/test_rag.py b/test/stdlib_intrinsics/test_rag/test_rag.py index 246ec3739..18985f029 100644 --- a/test/stdlib_intrinsics/test_rag/test_rag.py +++ b/test/stdlib_intrinsics/test_rag/test_rag.py @@ -86,11 +86,11 @@ def test_answerability(backend): context, next_user_turn, documents = _read_input_json("answerability.json") # First call triggers adapter loading - result = rag.check_answerability(context, next_user_turn, documents, backend) + result = rag.check_answerability(next_user_turn, documents, context, backend) assert pytest.approx(result) == 1.0 # Second call hits a different code path from the first one - result = rag.check_answerability(context, next_user_turn, documents, backend) + result = rag.check_answerability(next_user_turn, documents, context, backend) assert pytest.approx(result) == 1.0 @@ -104,11 +104,11 @@ def test_query_rewrite(backend): ) # First call triggers adapter loading - result = rag.rewrite_question(context, next_user_turn, backend) + result = rag.rewrite_question(next_user_turn, context, backend) assert result == expected # Second call hits a different code path from the first one - result = rag.rewrite_question(context, next_user_turn, backend) + result = rag.rewrite_question(next_user_turn, context, backend) assert result == expected @@ -119,11 +119,11 @@ def test_citations(backend): expected = _read_output_json("citations.json") # First call triggers adapter loading - result = rag.find_citations(context, assistant_response, docs, backend) + result = rag.find_citations(assistant_response, docs, context, backend) assert result == expected # Second call hits a different code path from the first one - result = rag.find_citations(context, assistant_response, docs, backend) + result = rag.find_citations(assistant_response, docs, context, backend) assert result == expected @@ -136,11 +136,11 @@ def test_context_relevance(backend): document = docs[0] # First call triggers adapter loading - result = rag.check_context_relevance(context, question, document, backend) + result = rag.check_context_relevance(question, document, context, backend) assert pytest.approx(result, abs=2e-2) == 0.45 # Second call hits a different code path from the first one - result = rag.check_context_relevance(context, question, document, backend) + result = rag.check_context_relevance(question, document, context, backend) assert pytest.approx(result, abs=2e-2) == 0.45 @@ -151,13 +151,13 @@ def test_hallucination_detection(backend): expected = _read_output_json("hallucination_detection.json") # First call triggers adapter loading - result = rag.flag_hallucinated_content(context, assistant_response, docs, backend) + result = rag.flag_hallucinated_content(assistant_response, docs, context, backend) # pytest.approx() chokes on lists of records, so we do this complicated dance. for r, e in zip(result, expected, strict=True): assert pytest.approx(r, abs=2e-2) == e # Second call hits a different code path from the first one - result = rag.flag_hallucinated_content(context, assistant_response, docs, backend) + result = rag.flag_hallucinated_content(assistant_response, docs, context, backend) for r, e in zip(result, expected, strict=True): assert pytest.approx(r, abs=2e-2) == e @@ -169,15 +169,15 @@ def test_answer_relevance(backend): expected_rewrite = "Alice, Bob, and Carol attended the meeting." # First call triggers adapter loading - result = rag.rewrite_answer_for_relevance(context, answer, docs, backend) + result = rag.rewrite_answer_for_relevance(answer, docs, context, backend) assert result == expected_rewrite # Second call hits a different code path from the first one - result = rag.rewrite_answer_for_relevance(context, answer, docs, backend) + result = rag.rewrite_answer_for_relevance(answer, docs, context, backend) assert result == expected_rewrite # Canned input always gets rewritten. Set threshold to disable the rewrite. - result = rag.rewrite_answer_for_relevance(context, answer, docs, backend, + result = rag.rewrite_answer_for_relevance(answer, docs, context, backend, rewrite_threshold=0.0) assert result == answer diff --git a/uv.lock b/uv.lock index 326315ab5..636e096fd 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and python_full_version < '4' and sys_platform == 'darwin'", @@ -850,7 +850,7 @@ name = "colorlog" version = "6.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "python_full_version < '3.14' and sys_platform == 'win32'" }, + { name = "colorama", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or (python_full_version >= '4' and sys_platform == 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/61/f083b5ac52e505dfc1c624eafbf8c7589a0d7f32daa398d2e7590efa5fda/colorlog-6.10.1.tar.gz", hash = "sha256:eb4ae5cb65fe7fec7773c2306061a8e63e02efc2c72eba9d27b0fa23c94f1321", size = 17162, upload-time = "2025-10-16T16:14:11.978Z" } wheels = [ @@ -1727,28 +1727,28 @@ wheels = [ ] [[package]] -name = "googleapis-common-protos" -version = "1.72.0" +name = "granite-common" +version = "0.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "protobuf" }, + { name = "jsonschema" }, + { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e5/7b/adfd75544c415c487b33061fe7ae526165241c1ea133f9a9125a56b39fd8/googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5", size = 147433, upload-time = "2025-11-06T18:29:24.087Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/9b/c0e846c0517c7581e63901f90f23a28aa6758ec2b686171a5f23ba73ae48/granite_common-0.3.2.tar.gz", hash = "sha256:f5bc850573700f160bab0ae921d5156a5fd4594ed6806ae82ddf8a6043aa4331", size = 273140, upload-time = "2025-10-24T19:29:51.449Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/ab/09169d5a4612a5f92490806649ac8d41e3ec9129c636754575b3553f4ea4/googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038", size = 297515, upload-time = "2025-11-06T18:29:13.14Z" }, + { url = "https://files.pythonhosted.org/packages/6b/54/cf965d50fe493f4fb8ab1d4dea371a5d9d74d8e0bfdf2bd17f41a3af973b/granite_common-0.3.2-py3-none-any.whl", hash = "sha256:45d5a99e264f9e009215daf039c85a1e1ea216983962bcff2c75b4fa4a815d87", size = 77490, upload-time = "2025-10-24T19:29:49.97Z" }, ] [[package]] -name = "granite-common" -version = "0.3.4" +name = "googleapis-common-protos" +version = "1.72.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jsonschema" }, - { name = "pydantic" }, + { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/39/7d/d03e3c55651476cb2838c6051457063d4f5fc448700054b73ef2edc28624/granite_common-0.3.4.tar.gz", hash = "sha256:5d362f1bd3e818ce2f6c280a51c730493b5a15b45d3d5bfdc4221bc856cf2b25", size = 273862, upload-time = "2025-11-12T01:17:15.285Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/7b/adfd75544c415c487b33061fe7ae526165241c1ea133f9a9125a56b39fd8/googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5", size = 147433, upload-time = "2025-11-06T18:29:24.087Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/18/715fa6eb7fabb06c7040b65e6ff8eb2bb787e266c75f1af9acbe5e3a1260/granite_common-0.3.4-py3-none-any.whl", hash = "sha256:85fdbf8b96fb667d6612959af863dc3cfd3a6c0744d9e3ad9aa52aee3e4a2909", size = 77557, upload-time = "2025-11-12T01:17:13.708Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ab/09169d5a4612a5f92490806649ac8d41e3ec9129c636754575b3553f4ea4/googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038", size = 297515, upload-time = "2025-11-06T18:29:13.14Z" }, ] [[package]] @@ -3327,10 +3327,10 @@ requires-dist = [ { name = "numpy", marker = "extra == 'vllm'", specifier = "<2.0.0" }, { name = "ollama", specifier = ">=0.5.1" }, { name = "openai" }, + { name = "peft", marker = "extra == 'hf'", git = "https://github.com/huggingface/peft.git?rev=293aea5df6db240856a77f89955d1a89ce38b50d" }, { name = "outlines", marker = "extra == 'hf'" }, { name = "outlines-core", marker = "extra == 'hf'", specifier = "==0.1.26" }, { name = "outlines-core", marker = "extra == 'vllm'", specifier = "==0.1.26" }, - { name = "peft", marker = "extra == 'hf'", git = "https://github.com/huggingface/peft.git?rev=293aea5df6db240856a77f89955d1a89ce38b50d" }, { name = "pillow" }, { name = "pydantic" }, { name = "requests", specifier = ">=2.32.3" }, @@ -4081,7 +4081,7 @@ name = "nvidia-cudnn-cu12" version = "9.5.1.17" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-cublas-cu12" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/2a/78/4535c9c7f859a64781e43c969a3a7e84c54634e319a996d43ef32ce46f83/nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2", size = 570988386, upload-time = "2024-10-25T19:54:26.39Z" }, @@ -4092,7 +4092,7 @@ name = "nvidia-cufft-cu12" version = "11.3.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-nvjitlink-cu12" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8f/16/73727675941ab8e6ffd86ca3a4b7b47065edcca7a997920b831f8147c99d/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ccba62eb9cef5559abd5e0d54ceed2d9934030f51163df018532142a8ec533e5", size = 200221632, upload-time = "2024-11-20T17:41:32.357Z" }, @@ -4121,9 +4121,9 @@ name = "nvidia-cusolver-cu12" version = "11.7.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, - { name = "nvidia-cusparse-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, - { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cusparse-cu12" }, + { name = "nvidia-nvjitlink-cu12" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f0/6e/c2cf12c9ff8b872e92b4a5740701e51ff17689c4d726fca91875b07f655d/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c", size = 158229790, upload-time = "2024-11-20T17:43:43.211Z" }, @@ -4135,7 +4135,7 @@ name = "nvidia-cusparse-cu12" version = "12.5.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-nvjitlink-cu12" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/06/1e/b8b7c2f4099a37b96af5c9bb158632ea9e5d9d27d7391d7eb8fc45236674/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7556d9eca156e18184b94947ade0fba5bb47d69cec46bf8660fd2c71a4b48b73", size = 216561367, upload-time = "2024-11-20T17:44:54.824Z" }, @@ -4180,9 +4180,9 @@ name = "ocrmac" version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "sys_platform == 'darwin'" }, - { name = "pillow", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin'" }, + { name = "click" }, + { name = "pillow" }, + { name = "pyobjc-framework-vision" }, ] sdist = { url = "https://files.pythonhosted.org/packages/dd/dc/de3e9635774b97d9766f6815bbb3f5ec9bce347115f10d9abbf2733a9316/ocrmac-1.0.0.tar.gz", hash = "sha256:5b299e9030c973d1f60f82db000d6c2e5ff271601878c7db0885e850597d1d2e", size = 1463997, upload-time = "2024-11-07T12:00:00.197Z" } wheels = [ @@ -4207,8 +4207,8 @@ name = "omegaconf" version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "antlr4-python3-runtime", version = "4.9.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, - { name = "pyyaml", marker = "python_full_version < '3.14'" }, + { name = "antlr4-python3-runtime", version = "4.9.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "pyyaml", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/09/48/6388f1bb9da707110532cb70ec4d2822858ddfb44f1cdf1233c20a80ea4b/omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7", size = 3298120, upload-time = "2022-12-08T20:59:22.753Z" } wheels = [ @@ -4239,7 +4239,7 @@ name = "opencv-python" version = "4.11.0.86" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "python_full_version < '3.14'" }, + { name = "numpy", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/17/06/68c27a523103dad5837dc5b87e71285280c4f098c60e4fe8a8db6486ab09/opencv-python-4.11.0.86.tar.gz", hash = "sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4", size = 95171956, upload-time = "2025-01-16T13:52:24.737Z" } wheels = [ @@ -5339,7 +5339,7 @@ name = "pyobjc-framework-cocoa" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-core" }, ] sdist = { url = "https://files.pythonhosted.org/packages/37/6f/89837da349fe7de6476c426f118096b147de923139556d98af1832c64b97/pyobjc_framework_cocoa-12.0.tar.gz", hash = "sha256:02d69305b698015a20fcc8e1296e1528e413d8cf9fdcd590478d359386d76e8a", size = 2771906, upload-time = "2025-10-21T08:30:51.765Z" } wheels = [ @@ -5357,8 +5357,8 @@ name = "pyobjc-framework-coreml" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0c/a0/875b5174794c984df60944be54df0282945f8bae4a606fbafa0c6b717ddd/pyobjc_framework_coreml-12.0.tar.gz", hash = "sha256:e1d7a9812886150881c86000fba885cb15201352c75fb286bd9e3a1819b5a4d5", size = 40814, upload-time = "2025-10-21T08:31:53.83Z" } wheels = [ @@ -5376,8 +5376,8 @@ name = "pyobjc-framework-quartz" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, ] sdist = { url = "https://files.pythonhosted.org/packages/91/0b/3c34fc9de790daff5ca49d1f36cb8dcc353ac10e4e29b4759e397a3831f4/pyobjc_framework_quartz-12.0.tar.gz", hash = "sha256:5bcb9e78d671447e04d89e2e3c39f3135157892243facc5f8468aa333e40d67f", size = 3159509, upload-time = "2025-10-21T08:40:01.918Z" } wheels = [ @@ -5395,10 +5395,10 @@ name = "pyobjc-framework-vision" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-coreml", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-core" }, + { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-framework-coreml" }, + { name = "pyobjc-framework-quartz" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/5a/07cdead5adb77d0742b014fa742d503706754e3ad10e39760e67bb58b497/pyobjc_framework_vision-12.0.tar.gz", hash = "sha256:942c9583f1d887ac9f704f3b0c21b3206b68e02852a87219db4309bb13a02f14", size = 59905, upload-time = "2025-10-21T08:41:53.741Z" } wheels = [ @@ -5856,17 +5856,17 @@ name = "rapidocr" version = "3.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorlog", marker = "python_full_version < '3.14'" }, - { name = "numpy", marker = "python_full_version < '3.14'" }, - { name = "omegaconf", marker = "python_full_version < '3.14'" }, - { name = "opencv-python", marker = "python_full_version < '3.14'" }, - { name = "pillow", marker = "python_full_version < '3.14'" }, - { name = "pyclipper", marker = "python_full_version < '3.14'" }, - { name = "pyyaml", marker = "python_full_version < '3.14'" }, - { name = "requests", marker = "python_full_version < '3.14'" }, - { name = "shapely", marker = "python_full_version < '3.14'" }, - { name = "six", marker = "python_full_version < '3.14'" }, - { name = "tqdm", marker = "python_full_version < '3.14'" }, + { name = "colorlog", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "numpy", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "omegaconf", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "opencv-python", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "pillow", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "pyclipper", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "pyyaml", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "requests", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "shapely", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "six", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "tqdm", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/3c/83/5b8c8075954c5b61d938b8954710d986134c4ca7c32a841ad7d8c844cf6c/rapidocr-3.4.2-py3-none-any.whl", hash = "sha256:17845fa8cc9a20a935111e59482f2214598bba1547000cfd960d8924dd4522a5", size = 15056674, upload-time = "2025-10-11T14:43:00.296Z" }, @@ -6806,7 +6806,7 @@ name = "shapely" version = "2.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "python_full_version < '4'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/bc/0989043118a27cccb4e906a46b7565ce36ca7b57f5a18b78f4f1b0f72d9d/shapely-2.1.2.tar.gz", hash = "sha256:2ed4ecb28320a433db18a5bf029986aa8afcfd740745e78847e330d5d94922a9", size = 315489, upload-time = "2025-09-24T13:51:41.432Z" } wheels = [ @@ -7563,7 +7563,7 @@ name = "triton" version = "3.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "setuptools", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "setuptools" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/76/04/d54d3a6d077c646624dc9461b0059e23fd5d30e0dbe67471e3654aec81f9/triton-3.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fad99beafc860501d7fcc1fb7045d9496cbe2c882b1674640304949165a916e7", size = 156441993, upload-time = "2025-04-09T20:27:25.107Z" }, @@ -8089,8 +8089,8 @@ name = "xformers" version = "0.0.30" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, - { name = "torch", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "numpy" }, + { name = "torch" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bf/f7/dd2269cce89fd1221947dd7cc3a60707ffe721ef55c1803ac3b1a1f7ae5c/xformers-0.0.30.tar.gz", hash = "sha256:a12bf3eb39e294cdbe8a7253ac9b665f41bac61d6d98df174e34ef7bdb6f2fc4", size = 10214139, upload-time = "2025-04-28T20:51:02.045Z" } wheels = [ From 59bf5a07db65e83b4da76ebb314a0e811dd2e5e1 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Thu, 13 Nov 2025 17:26:16 -0800 Subject: [PATCH 31/38] Fix errors introduced by merge Signed-off-by: Fred Reiss --- mellea/stdlib/intrinsics/rag.py | 5 +- pyproject.toml | 2 +- uv.lock | 102 ++++++++++++++++---------------- 3 files changed, 53 insertions(+), 56 deletions(-) diff --git a/mellea/stdlib/intrinsics/rag.py b/mellea/stdlib/intrinsics/rag.py index dadeed47f..c82ce7945 100644 --- a/mellea/stdlib/intrinsics/rag.py +++ b/mellea/stdlib/intrinsics/rag.py @@ -3,10 +3,7 @@ import collections.abc import json -import granite_common - -import mellea.stdlib.funcs as mfuncs -from mellea.backends import Backend +import mellea.stdlib.functional as mfuncs from mellea.backends.adapters.adapter import ( AdapterMixin, AdapterType, diff --git a/pyproject.toml b/pyproject.toml index 5ad91f08e..589db5c35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ "mistletoe>=1.4.0", "huggingface-hub>=0.33.4", "pillow", - "granite_common", # Needed for Intrinsics. + "granite-common>=0.3.4", # Needed for Intrinsics. "math_verify", # Needed for Majority Voting Sampling Strategies. # Needed for Majority Voting Sampling Strategies. "rouge_score", # Needed for Majority Voting Sampling Strategies. "llm-sandbox[docker]>=0.3.23", diff --git a/uv.lock b/uv.lock index 636e096fd..e097eee8c 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and python_full_version < '4' and sys_platform == 'darwin'", @@ -850,7 +850,7 @@ name = "colorlog" version = "6.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or (python_full_version >= '4' and sys_platform == 'win32')" }, + { name = "colorama", marker = "python_full_version < '3.14' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/61/f083b5ac52e505dfc1c624eafbf8c7589a0d7f32daa398d2e7590efa5fda/colorlog-6.10.1.tar.gz", hash = "sha256:eb4ae5cb65fe7fec7773c2306061a8e63e02efc2c72eba9d27b0fa23c94f1321", size = 17162, upload-time = "2025-10-16T16:14:11.978Z" } wheels = [ @@ -1727,28 +1727,28 @@ wheels = [ ] [[package]] -name = "granite-common" -version = "0.3.2" +name = "googleapis-common-protos" +version = "1.72.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jsonschema" }, - { name = "pydantic" }, + { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/70/9b/c0e846c0517c7581e63901f90f23a28aa6758ec2b686171a5f23ba73ae48/granite_common-0.3.2.tar.gz", hash = "sha256:f5bc850573700f160bab0ae921d5156a5fd4594ed6806ae82ddf8a6043aa4331", size = 273140, upload-time = "2025-10-24T19:29:51.449Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/7b/adfd75544c415c487b33061fe7ae526165241c1ea133f9a9125a56b39fd8/googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5", size = 147433, upload-time = "2025-11-06T18:29:24.087Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/54/cf965d50fe493f4fb8ab1d4dea371a5d9d74d8e0bfdf2bd17f41a3af973b/granite_common-0.3.2-py3-none-any.whl", hash = "sha256:45d5a99e264f9e009215daf039c85a1e1ea216983962bcff2c75b4fa4a815d87", size = 77490, upload-time = "2025-10-24T19:29:49.97Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ab/09169d5a4612a5f92490806649ac8d41e3ec9129c636754575b3553f4ea4/googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038", size = 297515, upload-time = "2025-11-06T18:29:13.14Z" }, ] [[package]] -name = "googleapis-common-protos" -version = "1.72.0" +name = "granite-common" +version = "0.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "protobuf" }, + { name = "jsonschema" }, + { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e5/7b/adfd75544c415c487b33061fe7ae526165241c1ea133f9a9125a56b39fd8/googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5", size = 147433, upload-time = "2025-11-06T18:29:24.087Z" } +sdist = { url = "https://files.pythonhosted.org/packages/39/7d/d03e3c55651476cb2838c6051457063d4f5fc448700054b73ef2edc28624/granite_common-0.3.4.tar.gz", hash = "sha256:5d362f1bd3e818ce2f6c280a51c730493b5a15b45d3d5bfdc4221bc856cf2b25", size = 273862, upload-time = "2025-11-12T01:17:15.285Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/ab/09169d5a4612a5f92490806649ac8d41e3ec9129c636754575b3553f4ea4/googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038", size = 297515, upload-time = "2025-11-06T18:29:13.14Z" }, + { url = "https://files.pythonhosted.org/packages/4f/18/715fa6eb7fabb06c7040b65e6ff8eb2bb787e266c75f1af9acbe5e3a1260/granite_common-0.3.4-py3-none-any.whl", hash = "sha256:85fdbf8b96fb667d6612959af863dc3cfd3a6c0744d9e3ad9aa52aee3e4a2909", size = 77557, upload-time = "2025-11-12T01:17:13.708Z" }, ] [[package]] @@ -3314,7 +3314,7 @@ requires-dist = [ { name = "datasets", marker = "extra == 'hf'", specifier = ">=4.0.0" }, { name = "docling", marker = "extra == 'docling'", specifier = ">=2.45.0" }, { name = "fastapi" }, - { name = "granite-common" }, + { name = "granite-common", specifier = ">=0.3.4" }, { name = "huggingface-hub", specifier = ">=0.33.4" }, { name = "ibm-watsonx-ai", marker = "extra == 'watsonx'", specifier = ">=1.3.31" }, { name = "jinja2" }, @@ -3327,10 +3327,10 @@ requires-dist = [ { name = "numpy", marker = "extra == 'vllm'", specifier = "<2.0.0" }, { name = "ollama", specifier = ">=0.5.1" }, { name = "openai" }, - { name = "peft", marker = "extra == 'hf'", git = "https://github.com/huggingface/peft.git?rev=293aea5df6db240856a77f89955d1a89ce38b50d" }, { name = "outlines", marker = "extra == 'hf'" }, { name = "outlines-core", marker = "extra == 'hf'", specifier = "==0.1.26" }, { name = "outlines-core", marker = "extra == 'vllm'", specifier = "==0.1.26" }, + { name = "peft", marker = "extra == 'hf'", git = "https://github.com/huggingface/peft.git?rev=293aea5df6db240856a77f89955d1a89ce38b50d" }, { name = "pillow" }, { name = "pydantic" }, { name = "requests", specifier = ">=2.32.3" }, @@ -4081,7 +4081,7 @@ name = "nvidia-cudnn-cu12" version = "9.5.1.17" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/2a/78/4535c9c7f859a64781e43c969a3a7e84c54634e319a996d43ef32ce46f83/nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2", size = 570988386, upload-time = "2024-10-25T19:54:26.39Z" }, @@ -4092,7 +4092,7 @@ name = "nvidia-cufft-cu12" version = "11.3.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8f/16/73727675941ab8e6ffd86ca3a4b7b47065edcca7a997920b831f8147c99d/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ccba62eb9cef5559abd5e0d54ceed2d9934030f51163df018532142a8ec533e5", size = 200221632, upload-time = "2024-11-20T17:41:32.357Z" }, @@ -4121,9 +4121,9 @@ name = "nvidia-cusolver-cu12" version = "11.7.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12" }, - { name = "nvidia-cusparse-cu12" }, - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-cusparse-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f0/6e/c2cf12c9ff8b872e92b4a5740701e51ff17689c4d726fca91875b07f655d/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c", size = 158229790, upload-time = "2024-11-20T17:43:43.211Z" }, @@ -4135,7 +4135,7 @@ name = "nvidia-cusparse-cu12" version = "12.5.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/06/1e/b8b7c2f4099a37b96af5c9bb158632ea9e5d9d27d7391d7eb8fc45236674/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7556d9eca156e18184b94947ade0fba5bb47d69cec46bf8660fd2c71a4b48b73", size = 216561367, upload-time = "2024-11-20T17:44:54.824Z" }, @@ -4180,9 +4180,9 @@ name = "ocrmac" version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "pillow" }, - { name = "pyobjc-framework-vision" }, + { name = "click", marker = "sys_platform == 'darwin'" }, + { name = "pillow", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/dd/dc/de3e9635774b97d9766f6815bbb3f5ec9bce347115f10d9abbf2733a9316/ocrmac-1.0.0.tar.gz", hash = "sha256:5b299e9030c973d1f60f82db000d6c2e5ff271601878c7db0885e850597d1d2e", size = 1463997, upload-time = "2024-11-07T12:00:00.197Z" } wheels = [ @@ -4207,8 +4207,8 @@ name = "omegaconf" version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "antlr4-python3-runtime", version = "4.9.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "pyyaml", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "antlr4-python3-runtime", version = "4.9.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, + { name = "pyyaml", marker = "python_full_version < '3.14'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/09/48/6388f1bb9da707110532cb70ec4d2822858ddfb44f1cdf1233c20a80ea4b/omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7", size = 3298120, upload-time = "2022-12-08T20:59:22.753Z" } wheels = [ @@ -4239,7 +4239,7 @@ name = "opencv-python" version = "4.11.0.86" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "numpy", marker = "python_full_version < '3.14'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/17/06/68c27a523103dad5837dc5b87e71285280c4f098c60e4fe8a8db6486ab09/opencv-python-4.11.0.86.tar.gz", hash = "sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4", size = 95171956, upload-time = "2025-01-16T13:52:24.737Z" } wheels = [ @@ -5339,7 +5339,7 @@ name = "pyobjc-framework-cocoa" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/37/6f/89837da349fe7de6476c426f118096b147de923139556d98af1832c64b97/pyobjc_framework_cocoa-12.0.tar.gz", hash = "sha256:02d69305b698015a20fcc8e1296e1528e413d8cf9fdcd590478d359386d76e8a", size = 2771906, upload-time = "2025-10-21T08:30:51.765Z" } wheels = [ @@ -5357,8 +5357,8 @@ name = "pyobjc-framework-coreml" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0c/a0/875b5174794c984df60944be54df0282945f8bae4a606fbafa0c6b717ddd/pyobjc_framework_coreml-12.0.tar.gz", hash = "sha256:e1d7a9812886150881c86000fba885cb15201352c75fb286bd9e3a1819b5a4d5", size = 40814, upload-time = "2025-10-21T08:31:53.83Z" } wheels = [ @@ -5376,8 +5376,8 @@ name = "pyobjc-framework-quartz" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/91/0b/3c34fc9de790daff5ca49d1f36cb8dcc353ac10e4e29b4759e397a3831f4/pyobjc_framework_quartz-12.0.tar.gz", hash = "sha256:5bcb9e78d671447e04d89e2e3c39f3135157892243facc5f8468aa333e40d67f", size = 3159509, upload-time = "2025-10-21T08:40:01.918Z" } wheels = [ @@ -5395,10 +5395,10 @@ name = "pyobjc-framework-vision" version = "12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core" }, - { name = "pyobjc-framework-cocoa" }, - { name = "pyobjc-framework-coreml" }, - { name = "pyobjc-framework-quartz" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-coreml", marker = "sys_platform == 'darwin'" }, + { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/5a/07cdead5adb77d0742b014fa742d503706754e3ad10e39760e67bb58b497/pyobjc_framework_vision-12.0.tar.gz", hash = "sha256:942c9583f1d887ac9f704f3b0c21b3206b68e02852a87219db4309bb13a02f14", size = 59905, upload-time = "2025-10-21T08:41:53.741Z" } wheels = [ @@ -5856,17 +5856,17 @@ name = "rapidocr" version = "3.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorlog", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "numpy", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "omegaconf", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "opencv-python", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "pillow", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "pyclipper", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "pyyaml", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "requests", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "shapely", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "six", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, - { name = "tqdm", marker = "python_full_version < '3.14' or python_full_version >= '4'" }, + { name = "colorlog", marker = "python_full_version < '3.14'" }, + { name = "numpy", marker = "python_full_version < '3.14'" }, + { name = "omegaconf", marker = "python_full_version < '3.14'" }, + { name = "opencv-python", marker = "python_full_version < '3.14'" }, + { name = "pillow", marker = "python_full_version < '3.14'" }, + { name = "pyclipper", marker = "python_full_version < '3.14'" }, + { name = "pyyaml", marker = "python_full_version < '3.14'" }, + { name = "requests", marker = "python_full_version < '3.14'" }, + { name = "shapely", marker = "python_full_version < '3.14'" }, + { name = "six", marker = "python_full_version < '3.14'" }, + { name = "tqdm", marker = "python_full_version < '3.14'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/3c/83/5b8c8075954c5b61d938b8954710d986134c4ca7c32a841ad7d8c844cf6c/rapidocr-3.4.2-py3-none-any.whl", hash = "sha256:17845fa8cc9a20a935111e59482f2214598bba1547000cfd960d8924dd4522a5", size = 15056674, upload-time = "2025-10-11T14:43:00.296Z" }, @@ -6806,7 +6806,7 @@ name = "shapely" version = "2.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "python_full_version < '4'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/bc/0989043118a27cccb4e906a46b7565ce36ca7b57f5a18b78f4f1b0f72d9d/shapely-2.1.2.tar.gz", hash = "sha256:2ed4ecb28320a433db18a5bf029986aa8afcfd740745e78847e330d5d94922a9", size = 315489, upload-time = "2025-09-24T13:51:41.432Z" } wheels = [ @@ -7563,7 +7563,7 @@ name = "triton" version = "3.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "setuptools" }, + { name = "setuptools", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/76/04/d54d3a6d077c646624dc9461b0059e23fd5d30e0dbe67471e3654aec81f9/triton-3.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fad99beafc860501d7fcc1fb7045d9496cbe2c882b1674640304949165a916e7", size = 156441993, upload-time = "2025-04-09T20:27:25.107Z" }, @@ -8089,8 +8089,8 @@ name = "xformers" version = "0.0.30" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, - { name = "torch" }, + { name = "numpy", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "torch", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bf/f7/dd2269cce89fd1221947dd7cc3a60707ffe721ef55c1803ac3b1a1f7ae5c/xformers-0.0.30.tar.gz", hash = "sha256:a12bf3eb39e294cdbe8a7253ac9b665f41bac61d6d98df174e34ef7bdb6f2fc4", size = 10214139, upload-time = "2025-04-28T20:51:02.045Z" } wheels = [ From 9ba28d19b9940b261d39bc15f7a219b47a8ec273 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Fri, 14 Nov 2025 09:23:43 -0800 Subject: [PATCH 32/38] Fix minor merge-induced issue Signed-off-by: Fred Reiss --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 589db5c35..13dcad4a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ "huggingface-hub>=0.33.4", "pillow", "granite-common>=0.3.4", # Needed for Intrinsics. - "math_verify", # Needed for Majority Voting Sampling Strategies. # Needed for Majority Voting Sampling Strategies. + "math_verify", # Needed for Majority Voting Sampling Strategies. "rouge_score", # Needed for Majority Voting Sampling Strategies. "llm-sandbox[docker]>=0.3.23", ] From 427ac2319d36f092df65f2553f73627584cd8779 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Fri, 14 Nov 2025 15:06:29 -0800 Subject: [PATCH 33/38] Remove repo_id parameter Signed-off-by: Fred Reiss --- mellea/backends/adapters/adapter.py | 54 +++++------ mellea/backends/adapters/catalog.py | 92 +++++++++++++++++++ mellea/backends/huggingface.py | 21 ++--- mellea/backends/openai.py | 17 +--- mellea/stdlib/intrinsics/intrinsic.py | 42 +++++---- mellea/stdlib/intrinsics/rag.py | 46 +++------- mellea/stdlib/requirement.py | 6 -- test/backends/test_adapters/test_adapter.py | 8 +- test/backends/test_huggingface.py | 25 ++--- .../test_openai_vllm/test_openai_vllm.py | 25 ++--- 10 files changed, 191 insertions(+), 145 deletions(-) create mode 100644 mellea/backends/adapters/catalog.py diff --git a/mellea/backends/adapters/adapter.py b/mellea/backends/adapters/adapter.py index 5da86715b..8f1577a86 100644 --- a/mellea/backends/adapters/adapter.py +++ b/mellea/backends/adapters/adapter.py @@ -2,7 +2,6 @@ import abc import pathlib -from enum import Enum from typing import Any, TypeVar import granite_common.intrinsics @@ -10,16 +9,10 @@ from litellm import cast from mellea.backends import Backend +from mellea.backends.adapters.catalog import AdapterType, fetch_intrinsic_metadata from mellea.backends.types import _ServerType -class AdapterType(Enum): - """Possible types of adapters for a backend.""" - - LORA = "lora" - ALORA = "alora" - - class Adapter(abc.ABC): """An adapter that can be added to a single backend.""" @@ -29,7 +22,8 @@ def __init__(self, name: str, adapter_type: AdapterType): Note: An adapter can only be added to a single backend. Args: - name: name of the adapter; when referencing this adapter, use adapter.qualified_name + name: name of the adapter; when referencing this adapter, use + adapter.qualified_name adapter_type: enum describing what type of adapter it is (ie LORA / ALORA) """ self.name = name @@ -78,11 +72,10 @@ def get_local_hf_path(self, base_model_name: str) -> str: class GraniteCommonAdapter(OpenAIAdapter, LocalHFAdapter): - """Adapter for intrinsics that utilize the GraniteCommon library.""" + """Adapter for intrinsics that utilize the ``granite-common`` library.""" def __init__( self, - repo_id: str, intrinsic_name: str, adapter_type: AdapterType = AdapterType.ALORA, config_file: str | pathlib.Path | None = None, @@ -95,9 +88,6 @@ def __init__( Most intrinsics support LoRA or aLoRA adapter types. Args: - repo_id: Name of Hugging Face Hub repository containing the adapters that - implement the intrinsic; for example, - "generative-computing/rag-intrinsics-lib" intrinsic_name: name of the intrinsic; the local name of the loaded adapter that implements this intrinsic will be adapter.qualified_name adapter_type: enum describing what type of adapter it is (ie LORA / ALORA) @@ -106,15 +96,20 @@ def __init__( base_model_name: optional; if provided with no config_file/config_dict, will be used to look up the granite_common config for this adapter """ - assert adapter_type in (AdapterType.ALORA, AdapterType.LORA), ( - f"{adapter_type} not supported" - ) - super().__init__(f"{repo_id}_{intrinsic_name}", adapter_type) + super().__init__(intrinsic_name, adapter_type) - self.repo_id = repo_id self.intrinsic_name = intrinsic_name + self.intrinsic_metadata = fetch_intrinsic_metadata(intrinsic_name) self.base_model_name = base_model_name + if adapter_type not in self.intrinsic_metadata.adapter_types: + raise ValueError( + f"Intrinsic '{intrinsic_name}' not available as an adapter of type " + f"'{adapter_type}. Available types are " + f"{self.intrinsic_metadata.adapter_types}." + ) + self.adapter_type = adapter_type + # If any of the optional params are specified, attempt to set up the # config for the intrinsic here. if config_file and config_dict: @@ -132,12 +127,16 @@ def __init__( assert self.base_model_name is not None, ( "must provide `base_model_name` if not providing a `config_file` or `config_dict`" ) + # We're converting the adapter type to a boolean flag here. + assert adapter_type in (AdapterType.ALORA, AdapterType.LORA), ( + f"{adapter_type} not supported" + ) is_alora = self.adapter_type == AdapterType.ALORA config_file = granite_common.intrinsics.obtain_io_yaml( self.intrinsic_name, self.base_model_name, alora=is_alora, - repo_id=repo_id, + repo_id=self.intrinsic_metadata.repo_id, ) if config_file: with open(config_file, encoding="utf-8") as f: @@ -183,15 +182,17 @@ def get_local_hf_path(self, base_model_name: str) -> str: """Returns the path needed to load the adapter. Args: - base_model_name: the base model; typically the last part of the huggingface model id like "granite-3.3-8b-instruct" + base_model_name: the base model; typically the last part of the huggingface + model id like "granite-3.3-8b-instruct" """ return self.download_and_get_path(base_model_name) def download_and_get_path(self, base_model_name: str) -> str: - """Downloads the required rag intrinsics files if necessary and returns the path to the them. + """Downloads the required rag intrinsics files if necessary and returns the path to them. Args: - base_model_name: the base model; typically the last part of the huggingface model id like "granite-3.3-8b-instruct" + base_model_name: the base model; typically the last part of the huggingface + model id like "granite-3.3-8b-instruct" Returns: a path to the files @@ -202,7 +203,7 @@ def download_and_get_path(self, base_model_name: str) -> str: self.intrinsic_name, base_model_name, alora=is_alora, - repo_id=self.repo_id, + repo_id=self.intrinsic_metadata.repo_id, ) ) @@ -216,9 +217,8 @@ def get_path_on_remote(self, base_model_name: str, base_path: str) -> str: def get_adapter_for_intrinsic( - repo_id: str, intrinsic_name: str, - intrinsic_adapter_types: list[AdapterType], + intrinsic_adapter_types: list[AdapterType] | tuple[AdapterType, ...], available_adapters: dict[str, T], ) -> T | None: """Finds an adapter from a dict of available adapters based on the intrinsic name and its allowed adapter types. @@ -235,7 +235,7 @@ def get_adapter_for_intrinsic( """ adapter = None for adapter_type in intrinsic_adapter_types: - qualified_name = f"{repo_id}_{intrinsic_name}_{adapter_type.value}" + qualified_name = f"{intrinsic_name}_{adapter_type.value}" adapter = available_adapters.get(qualified_name) if adapter is not None: break diff --git a/mellea/backends/adapters/catalog.py b/mellea/backends/adapters/catalog.py new file mode 100644 index 000000000..0ef73219e --- /dev/null +++ b/mellea/backends/adapters/catalog.py @@ -0,0 +1,92 @@ +"""Catalog of available intrinsics. + +Catalog of intrinsics currently known to Mellea,including metadata about where to find +LoRA and aLoRA adapters that implement said intrinsics. +""" + +import enum + +import pydantic + + +class AdapterType(enum.Enum): + """Possible types of adapters for a backend.""" + + LORA = "lora" + ALORA = "alora" + + +class IntriniscsCatalogEntry(pydantic.BaseModel): + """A single row in the main intrinsics catalog table. + + We use Pydantic for this dataclass because the rest of Mellea also uses Pydantic. + """ + + name: str = pydantic.Field(description="User-visible name of the intrinsic.") + internal_name: str | None = pydantic.Field( + default=None, + description="Internal name used for adapter loading, or None if the name used " + "for that purpose is the same as self.name", + ) + repo_id: str = pydantic.Field( + description="Hugging Face repository (aka 'model') where adapters for the " + "intrinsic are located." + ) + adapter_types: tuple[AdapterType, ...] = pydantic.Field( + default=(AdapterType.LORA, AdapterType.ALORA), + description="Adapter types that are known to be available for this intrinsic.", + ) + + +_RAG_REPO = "ibm-granite/rag-intrinsics-lib" + + +_INTRINSICS_CATALOG_ENTRIES = [ + ############################################ + # Core Intrinsics + ############################################ + IntriniscsCatalogEntry(name="requirement_check", repo_id=_RAG_REPO), + IntriniscsCatalogEntry(name="uncertainty", repo_id=_RAG_REPO), + ############################################ + # RAG Intrinsics + ############################################ + IntriniscsCatalogEntry( + name="answer_relevance_classifier", + repo_id=_RAG_REPO, + adapter_types=(AdapterType.LORA,), + ), + IntriniscsCatalogEntry(name="answer_relevance_rewriter", repo_id=_RAG_REPO), + IntriniscsCatalogEntry(name="answerability", repo_id=_RAG_REPO), + IntriniscsCatalogEntry(name="citations", repo_id=_RAG_REPO), + IntriniscsCatalogEntry(name="context_relevance", repo_id=_RAG_REPO), + IntriniscsCatalogEntry(name="hallucination_detection", repo_id=_RAG_REPO), + IntriniscsCatalogEntry(name="query_rewrite", repo_id=_RAG_REPO), +] + +_INTRINSICS_CATALOG = {e.name: e for e in _INTRINSICS_CATALOG_ENTRIES} +"""Catalog of intrinsics that Mellea knows about. + +Mellea code should access this catalog via :func:`fetch_intrinsic_metadata()`""" + + +def known_intrinsic_names() -> list[str]: + """:returns: List of all known user-visible names for intrinsics.""" + return list(_INTRINSICS_CATALOG.keys()) + + +def fetch_intrinsic_metadata(intrinsic_name: str) -> IntriniscsCatalogEntry: + """Retrieve information about the adapter that backs an intrinsic. + + :param intrinsic_name: User-visible name of the intrinsic + + :returns: Metadata about the adapter(s) that implement the intrinsic. + """ + if intrinsic_name not in _INTRINSICS_CATALOG: + raise ValueError( + f"Unknown intrinsic name '{intrinsic_name}'. Valid names are " + f"{known_intrinsic_names()}" + ) + + # Make a copy in case some naughty downstream code decides to modify the returned + # value. + return _INTRINSICS_CATALOG[intrinsic_name].model_copy() diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index 39b7f1b36..286dac024 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -41,6 +41,7 @@ LocalHFAdapter, get_adapter_for_intrinsic, ) +from mellea.backends.adapters.catalog import fetch_intrinsic_metadata from mellea.backends.cache import Cache, SimpleLRUCache from mellea.backends.formatter import Formatter, FormatterBackend, TemplateFormatter from mellea.backends.model_ids import ModelIdentifier @@ -65,12 +66,7 @@ ) from mellea.stdlib.chat import Message from mellea.stdlib.intrinsics.intrinsic import Intrinsic -from mellea.stdlib.requirement import ( - REQUIREMENT_REPO_ID, - ALoraRequirement, - LLMaJRequirement, - Requirement, -) +from mellea.stdlib.requirement import ALoraRequirement, LLMaJRequirement, Requirement assert outlines, "outlines needs to be present to make outlines_core work" @@ -215,12 +211,10 @@ def generate_from_context( ) alora_action = ALoraRequirement(action.description, adapter_name) - # Check if a requirement_check (or AloraRequirement specified) adapter exists. + # Check if a requirement_check (or AloraRequirement specified) adapter + # exists. alora_req_adapter = get_adapter_for_intrinsic( - REQUIREMENT_REPO_ID, - adapter_name, - [AdapterType.ALORA], - self._added_adapters, + adapter_name, [AdapterType.ALORA], self._added_adapters ) if alora_req_adapter is None: # Log a warning if using an AloraRequirement but no adapter fit. @@ -285,10 +279,7 @@ def _generate_from_intrinsic( del model_options[ModelOption.STREAM] adapter = get_adapter_for_intrinsic( - action.repo_id, - action.intrinsic_name, - action.adapter_types, - self._added_adapters, + action.intrinsic_name, action.adapter_types, self._added_adapters ) if adapter is None: raise ValueError( diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index da3a2fd84..f7109d489 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -57,12 +57,7 @@ ) from mellea.stdlib.chat import Message from mellea.stdlib.intrinsics.intrinsic import Intrinsic -from mellea.stdlib.requirement import ( - REQUIREMENT_REPO_ID, - ALoraRequirement, - LLMaJRequirement, - Requirement, -) +from mellea.stdlib.requirement import ALoraRequirement, LLMaJRequirement, Requirement if TYPE_CHECKING: from transformers.tokenization_utils import PreTrainedTokenizer @@ -339,10 +334,7 @@ def generate_from_chat_context( # Check if a requirement_check (or AloraRequirement specified) adapter exists. alora_req_adapter = get_adapter_for_intrinsic( - REQUIREMENT_REPO_ID, - adapter_name, - [AdapterType.ALORA], - self._added_adapters, + adapter_name, [AdapterType.ALORA], self._added_adapters ) if alora_req_adapter is None: # Log a warning if using an AloraRequirement but no adapter fit. @@ -415,10 +407,7 @@ def _generate_from_intrinsic( del model_opts[ModelOption.STREAM] adapter = get_adapter_for_intrinsic( - action.repo_id, - action.intrinsic_name, - action.adapter_types, - self._added_adapters, + action.intrinsic_name, action.adapter_types, self._added_adapters ) if adapter is None: raise ValueError( diff --git a/mellea/stdlib/intrinsics/intrinsic.py b/mellea/stdlib/intrinsics/intrinsic.py index f17b1f762..4c54a55dd 100644 --- a/mellea/stdlib/intrinsics/intrinsic.py +++ b/mellea/stdlib/intrinsics/intrinsic.py @@ -4,7 +4,7 @@ from copy import copy from typing import cast -from mellea.backends.adapters.adapter import AdapterType +from mellea.backends.adapters.catalog import AdapterType, fetch_intrinsic_metadata from mellea.stdlib.base import CBlock, Component, TemplateRepresentation @@ -12,11 +12,7 @@ class Intrinsic(Component): """A component representing an intrinsic.""" def __init__( - self, - repo_id: str, - intrinsic_name: str, - intrinsic_kwargs: dict | None = None, - adapter_types: list[AdapterType] = [AdapterType.ALORA, AdapterType.LORA], + self, intrinsic_name: str, intrinsic_kwargs: dict | None = None ) -> None: """A component for rewriting messages using intrinsics. @@ -29,23 +25,26 @@ def __init__( An intrinsic component should correspond to a loaded adapter. Args: - repo_id: name of Hugging Face Hub repository containing adapters that - implement the intrinsic - intrinsic_name: the name of the intrinsic; must match the name of the - associated adapters within the target repository - intrinsic_kwargs: some intrinsics require kwargs when utilizing them; provide them here - adapter_types: list of adapter types that can be used for this intrinsic + intrinsic_name: the user-visible name of the intrinsic; must match a known + name in Mellea's intrinsics catalog. + intrinsic_kwargs: some intrinsics require kwargs when utilizing them; + provide them here """ - self.repo_id = repo_id - self.intrinsic_name = intrinsic_name - - # Copy the list so that this intrinsic has its own list that can be modified independently. - self.adapter_types = copy(adapter_types) - + self.metadata = fetch_intrinsic_metadata(intrinsic_name) if intrinsic_kwargs is None: intrinsic_kwargs = {} self.intrinsic_kwargs = intrinsic_kwargs + @property + def intrinsic_name(self): + """User-visible name of this intrinsic.""" + return self.metadata.name + + @property + def adapter_types(self) -> tuple[AdapterType, ...]: + """Tuple of available adapter types that implement this intrinsic.""" + return self.metadata.adapter_types + def parts(self) -> list[Component | CBlock]: """The set of all the constituent parts of the `Intrinsic`. @@ -55,10 +54,13 @@ def parts(self) -> list[Component | CBlock]: raise NotImplementedError("parts isn't implemented by default") def format_for_llm(self) -> TemplateRepresentation | str: - """`Intrinsic` doesn't implement `format_for_default`. Formats the `Intrinsic` into a `TemplateRepresentation` or string. + """`Intrinsic` doesn't implement `format_for_default`. + + Formats the `Intrinsic` into a `TemplateRepresentation` or string. Returns: a `TemplateRepresentation` or string """ raise NotImplementedError( - "`Intrinsic` doesn't implement format_for_llm by default. You should only use an `Intrinsic` as the action and not as a part of the context." + "`Intrinsic` doesn't implement format_for_llm by default. You should only " + "use an `Intrinsic` as the action and not as a part of the context." ) diff --git a/mellea/stdlib/intrinsics/rag.py b/mellea/stdlib/intrinsics/rag.py index c82ce7945..5885c9004 100644 --- a/mellea/stdlib/intrinsics/rag.py +++ b/mellea/stdlib/intrinsics/rag.py @@ -13,28 +13,20 @@ from mellea.stdlib.chat import Message from mellea.stdlib.intrinsics.intrinsic import Intrinsic -RAG_REPO = "ibm-granite/rag-intrinsics-lib" -# RAG_REPO = "generative-computing/rag-intrinsics-lib" - -# Mapping from name to . -_CATALOG = { - "answer_relevance_classifier": (RAG_REPO, AdapterType.LORA), - "answer_relevance_rewriter": (RAG_REPO, AdapterType.LORA), - "answerability": (RAG_REPO, AdapterType.LORA), - "citations": (RAG_REPO, AdapterType.LORA), - "context_relevance": (RAG_REPO, AdapterType.LORA), - "hallucination_detection": (RAG_REPO, AdapterType.LORA), - "query_rewrite": (RAG_REPO, AdapterType.LORA), -} -ANSWERABILITY_MODEL_NAME = "answerability" - _ANSWER_RELEVANCE_CORRECTION_METHODS = { - "Excessive unnecessary information": "removing the excessive information from the draft response", - "Unduly restrictive": "providing answer without the unwarranted restriction, or indicating that the desired answer is not available", - "Too vague or generic": "providing more crisp and to-the-point answer, or indicating that the desired answer is not available", - "Contextual misalignment": "providing a response that answers the last user inquiry, taking into account the context of the conversation", - "Misinterpreted inquiry": "providing answer only to the correct interpretation of the inquiry, or attempting clarification if the inquiry is ambiguous or otherwise confusing, or indicating that the desired answer is not available", - "No attempt": "providing a relevant response if an inquiry should be answered, or providing a short response if the last user utterance contains no inquiry", + "Excessive unnecessary information": "removing the excessive information from the " + "draft response", + "Unduly restrictive": "providing answer without the unwarranted restriction, or " + "indicating that the desired answer is not available", + "Too vague or generic": "providing more crisp and to-the-point answer, or " + "indicating that the desired answer is not available", + "Contextual misalignment": "providing a response that answers the last user " + "inquiry, taking into account the context of the conversation", + "Misinterpreted inquiry": "providing answer only to the correct interpretation of " + "the inquiry, or attempting clarification if the inquiry is ambiguous or otherwise " + "confusing, or indicating that the desired answer is not available", + "No attempt": "providing a relevant response if an inquiry should be answered, or " + "providing a short response if the last user utterance contains no inquiry", } """Prompting strings for the answer relevance rewriter. This model is a (a)LoRA adapter, so it's important to stick to in-domain prompts.""" @@ -54,25 +46,17 @@ def _call_intrinsic( # Adapter needs to be present in the backend before it can be invoked. # We must create the Adapter object in order to determine whether we need to create # the Adapter object. - if intrinsic_name not in _CATALOG: - raise ValueError( - f"Unexpected intrinsic name '{intrinsic_name}'. " - f"Should be one of {list(_CATALOG.keys())}" - ) - repo_id, adapter_type = _CATALOG[intrinsic_name] base_model_name = backend.base_model_name if base_model_name is None: raise ValueError("Backend has no model ID") adapter = GraniteCommonAdapter( - repo_id, intrinsic_name, adapter_type, base_model_name=base_model_name + intrinsic_name, adapter_type=AdapterType.LORA, base_model_name=base_model_name ) if adapter.qualified_name not in backend.list_adapters(): backend.add_adapter(adapter) # Create the AST node for the action we wish to perform. - intrinsic = Intrinsic( - repo_id, intrinsic_name, adapter_types=[adapter_type], intrinsic_kwargs=kwargs - ) + intrinsic = Intrinsic(intrinsic_name, intrinsic_kwargs=kwargs) # Execute the AST node. model_output_thunk, _ = mfuncs.act( diff --git a/mellea/stdlib/requirement.py b/mellea/stdlib/requirement.py index 954f51971..4cef5e35f 100644 --- a/mellea/stdlib/requirement.py +++ b/mellea/stdlib/requirement.py @@ -19,10 +19,6 @@ ) from mellea.stdlib.intrinsics.intrinsic import Intrinsic -REQUIREMENT_REPO_ID = "ibm-granite/rag-intrinsics-lib" -"""Hard-coded repository on Hugging Face Hub where Mellea keeps its requirement -intrinsic's aLoRA adapter.""" - def default_output_to_bool(x: CBlock | str) -> bool: """Checks if a given output should be marked converted to `True`. @@ -234,10 +230,8 @@ def __init__(self, description: str, intrinsic_name: str | None = None): # Initialize the other side of the inheritance tree Intrinsic.__init__( self, - repo_id=REQUIREMENT_REPO_ID, intrinsic_name=intrinsic_name, intrinsic_kwargs={"requirement": f"{self.description}"}, - adapter_types=[AdapterType.ALORA], ) diff --git a/test/backends/test_adapters/test_adapter.py b/test/backends/test_adapters/test_adapter.py index 1d03ba00c..f5abbb846 100644 --- a/test/backends/test_adapters/test_adapter.py +++ b/test/backends/test_adapters/test_adapter.py @@ -3,18 +3,18 @@ from mellea.backends.adapters.adapter import GraniteCommonAdapter + # The backend tests handle most of the adapter testing. Do a basic test here # to make sure init and config loading work. def test_adapter_init(): dir_file = pathlib.Path(__file__).parent.joinpath("intrinsics-data") answerability_file = f"{dir_file}/answerability.yaml" - adapter = GraniteCommonAdapter( - "ibm-granite/rag-intrinsics-lib", - "answerability", config_file=answerability_file) - + adapter = GraniteCommonAdapter("answerability", config_file=answerability_file) + assert adapter.config is not None assert adapter.config["parameters"]["max_completion_tokens"] == 6 + if __name__ == "__main__": pytest.main([__file__]) diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index 6573e1ae4..f0854066d 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -1,4 +1,5 @@ import asyncio + import pydantic import pytest from typing_extensions import Annotated @@ -9,21 +10,11 @@ from mellea.backends.formatter import TemplateFormatter from mellea.backends.huggingface import LocalHFBackend from mellea.backends.types import ModelOption -from mellea.stdlib.base import ( - CBlock, - ChatContext, - Context, - ModelOutputThunk, - SimpleContext, -) -from mellea.stdlib.requirement import ( - ALoraRequirement, - LLMaJRequirement, - Requirement, - ValidationResult, - default_output_to_bool, - REQUIREMENT_REPO_ID, -) +from mellea.stdlib.base import (CBlock, ChatContext, Context, ModelOutputThunk, + SimpleContext) +from mellea.stdlib.requirement import (ALoraRequirement, LLMaJRequirement, + Requirement, ValidationResult, + default_output_to_bool) @pytest.fixture(scope="module") @@ -36,7 +27,7 @@ def backend(): ) backend.add_adapter( GraniteCommonAdapter( - REQUIREMENT_REPO_ID, "requirement_check", base_model_name=backend.base_model_name + "requirement_check", base_model_name=backend.base_model_name ) ) return backend @@ -53,7 +44,7 @@ def session(backend): def test_adapters(backend): assert len(backend._added_adapters.items()) > 0 - expected_qualified_name = f"{REQUIREMENT_REPO_ID}_requirement_check_alora" + expected_qualified_name = "requirement_check_alora" adapter = backend._added_adapters[expected_qualified_name] backend.load_adapter(adapter.qualified_name) assert adapter.qualified_name in backend._loaded_adapters diff --git a/test/backends/test_openai_vllm/test_openai_vllm.py b/test/backends/test_openai_vllm/test_openai_vllm.py index 099bf0846..8fc05a1b0 100644 --- a/test/backends/test_openai_vllm/test_openai_vllm.py +++ b/test/backends/test_openai_vllm/test_openai_vllm.py @@ -1,16 +1,18 @@ # test/rits_backend_tests/test_openai_integration.py -from mellea import MelleaSession -from mellea.backends.adapters.adapter import GraniteCommonAdapter -from mellea.stdlib.base import CBlock, ModelOutputThunk, ChatContext, Context -from mellea.backends.openai import OpenAIBackend -from mellea.stdlib.requirement import REQUIREMENT_REPO_ID, Requirement, ALoraRequirement, LLMaJRequirement, req -from mellea.backends.formatter import TemplateFormatter -from mellea.backends.types import _ServerType, ModelOption +import os import pydantic -from typing_extensions import Annotated import pytest -import os +from typing_extensions import Annotated + +from mellea import MelleaSession +from mellea.backends.adapters.adapter import GraniteCommonAdapter +from mellea.backends.formatter import TemplateFormatter +from mellea.backends.openai import OpenAIBackend +from mellea.backends.types import ModelOption, _ServerType +from mellea.stdlib.base import CBlock, ChatContext, Context, ModelOutputThunk +from mellea.stdlib.requirement import (ALoraRequirement, LLMaJRequirement, + Requirement, req) # The vllm tests are disabled by default, because we need a test environment with the vLLM server running. # We use an env var VLLM_TESTS_ENABLED to enable these tests. @@ -135,14 +137,15 @@ class TestOpenAIALoraStuff: base_url="http://localhost:8000/v1", api_key="EMPTY", ) - backend.add_adapter(GraniteCommonAdapter(REQUIREMENT_REPO_ID, "requirement_check", base_model_name=backend.base_model_name)) + backend.add_adapter(GraniteCommonAdapter("requirement_check", + base_model_name=backend.base_model_name)) m = MelleaSession(backend, ctx=ChatContext()) def test_adapters(self): assert len(self.backend._added_adapters.items()) > 0 - adapter = self.backend._added_adapters[f"{REQUIREMENT_REPO_ID}_requirement_check_alora"] + adapter = self.backend._added_adapters["requirement_check_alora"] self.backend.load_adapter(adapter.qualified_name) assert adapter.qualified_name in self.backend._loaded_adapters From 2a393b20d6e913f2e2e0102b553606b1c8b038cb Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Fri, 14 Nov 2025 17:32:37 -0800 Subject: [PATCH 34/38] Upgrade to latest peft and granite-common Signed-off-by: Fred Reiss --- pyproject.toml | 3 +-- uv.lock | 10 +++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 13dcad4a6..cd0e3a707 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,8 +73,7 @@ hf = [ "datasets>=4.0.0", "outlines-core==0.1.26", "outlines", # intentionally un-versioned, expecting a minor update. coutlines-core version should be enough to specify it - # "peft>=0.17.2", # This package can be re-enabled once peft 0.17.2 has been released and below commit is confirmed to be a part of it. - "peft @ git+https://github.com/huggingface/peft.git@293aea5df6db240856a77f89955d1a89ce38b50d", + "peft>=0.18.0", # aLoRA support was added in Peft 0.18.0 "transformers>=4.53.2", "trl==0.19.1", ] diff --git a/uv.lock b/uv.lock index e097eee8c..a7459d619 100644 --- a/uv.lock +++ b/uv.lock @@ -3330,7 +3330,7 @@ requires-dist = [ { name = "outlines", marker = "extra == 'hf'" }, { name = "outlines-core", marker = "extra == 'hf'", specifier = "==0.1.26" }, { name = "outlines-core", marker = "extra == 'vllm'", specifier = "==0.1.26" }, - { name = "peft", marker = "extra == 'hf'", git = "https://github.com/huggingface/peft.git?rev=293aea5df6db240856a77f89955d1a89ce38b50d" }, + { name = "peft", marker = "extra == 'hf'", specifier = ">=0.18.0" }, { name = "pillow" }, { name = "pydantic" }, { name = "requests", specifier = ">=2.32.3" }, @@ -4607,8 +4607,8 @@ wheels = [ [[package]] name = "peft" -version = "0.17.2.dev0" -source = { git = "https://github.com/huggingface/peft.git?rev=293aea5df6db240856a77f89955d1a89ce38b50d#293aea5df6db240856a77f89955d1a89ce38b50d" } +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "accelerate" }, { name = "huggingface-hub" }, @@ -4621,6 +4621,10 @@ dependencies = [ { name = "tqdm" }, { name = "transformers" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/4b/0c/f2938db546ac7fc961ab5917cd50fcf5d0d70b406de93e3faccaa504e152/peft-0.18.0.tar.gz", hash = "sha256:c81c80b2056ab40c23d58ef25f74daab417ac653970718589a11a8af28218588", size = 634141, upload-time = "2025-11-13T11:13:06.603Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/55/481bf25613d40ef53534f664deba7b138fe566356b6ca10304e2b3b2529c/peft-0.18.0-py3-none-any.whl", hash = "sha256:624f69ca6393b765ccc6734adda7ca57d80b238f0900a42c357d8b67a03d62ff", size = 556427, upload-time = "2025-11-13T11:13:03.664Z" }, +] [[package]] name = "pexpect" From 4a89e8af410701a445f7ca22de74d9325eac4b18 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Fri, 14 Nov 2025 17:42:39 -0800 Subject: [PATCH 35/38] Upgrade dependencies again --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cd0e3a707..052505257 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ "mistletoe>=1.4.0", "huggingface-hub>=0.33.4", "pillow", - "granite-common>=0.3.4", # Needed for Intrinsics. + "granite-common>=0.3.5", # Needed for Intrinsics. "math_verify", # Needed for Majority Voting Sampling Strategies. "rouge_score", # Needed for Majority Voting Sampling Strategies. "llm-sandbox[docker]>=0.3.23", diff --git a/uv.lock b/uv.lock index a7459d619..ee281b4a6 100644 --- a/uv.lock +++ b/uv.lock @@ -1740,15 +1740,15 @@ wheels = [ [[package]] name = "granite-common" -version = "0.3.4" +version = "0.3.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonschema" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/39/7d/d03e3c55651476cb2838c6051457063d4f5fc448700054b73ef2edc28624/granite_common-0.3.4.tar.gz", hash = "sha256:5d362f1bd3e818ce2f6c280a51c730493b5a15b45d3d5bfdc4221bc856cf2b25", size = 273862, upload-time = "2025-11-12T01:17:15.285Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/b8/cba7a2399079838f793cc138c5b341df965c82e7cdaaf4c37deeffa0a14c/granite_common-0.3.5.tar.gz", hash = "sha256:80d4251b9294b6ec234d5aa4e273801b66f7cc4c5bc77151e5c22e7d7f5a19cd", size = 273710, upload-time = "2025-11-15T01:32:38.761Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/18/715fa6eb7fabb06c7040b65e6ff8eb2bb787e266c75f1af9acbe5e3a1260/granite_common-0.3.4-py3-none-any.whl", hash = "sha256:85fdbf8b96fb667d6612959af863dc3cfd3a6c0744d9e3ad9aa52aee3e4a2909", size = 77557, upload-time = "2025-11-12T01:17:13.708Z" }, + { url = "https://files.pythonhosted.org/packages/4c/f4/79d121e4192cf7871122995f060f891c6dbcb91b9b4753e4dd27852704c4/granite_common-0.3.5-py3-none-any.whl", hash = "sha256:fca8fdb7caff7f5714bfda9c81438f4a6df974e0b01631421cd6ca1a19bbb07e", size = 77551, upload-time = "2025-11-15T01:32:37.186Z" }, ] [[package]] @@ -3314,7 +3314,7 @@ requires-dist = [ { name = "datasets", marker = "extra == 'hf'", specifier = ">=4.0.0" }, { name = "docling", marker = "extra == 'docling'", specifier = ">=2.45.0" }, { name = "fastapi" }, - { name = "granite-common", specifier = ">=0.3.4" }, + { name = "granite-common", specifier = ">=0.3.5" }, { name = "huggingface-hub", specifier = ">=0.33.4" }, { name = "ibm-watsonx-ai", marker = "extra == 'watsonx'", specifier = ">=1.3.31" }, { name = "jinja2" }, From 628a7d025b715114c4516967119311cf014ae2ec Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Fri, 14 Nov 2025 17:55:18 -0800 Subject: [PATCH 36/38] Fix broken test Signed-off-by: Fred Reiss --- test/backends/test_litellm_ollama.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/backends/test_litellm_ollama.py b/test/backends/test_litellm_ollama.py index d5acc79c1..5c1ddf86f 100644 --- a/test/backends/test_litellm_ollama.py +++ b/test/backends/test_litellm_ollama.py @@ -8,6 +8,10 @@ from mellea.stdlib.base import CBlock, SimpleContext from mellea.stdlib.chat import Message from mellea.stdlib.sampling import RejectionSamplingStrategy +from mellea.backends import model_ids + + +_MODEL_ID = f"ollama_chat/{model_ids.IBM_GRANITE_4_MICRO_3B.ollama_name}" @pytest.fixture(scope="function") @@ -22,12 +26,12 @@ def backend(gh_run: int): url = url.replace("127.0.0.1", "http://localhost") return LiteLLMBackend( - model_id="ollama_chat/llama3.2:1b", + model_id=_MODEL_ID, base_url=url, model_options={"api_base": url}, ) else: - return LiteLLMBackend() + return LiteLLMBackend(model_id=_MODEL_ID) @pytest.fixture(scope="function") @@ -106,9 +110,13 @@ def test_litellm_ollama_instruct_options(session): model_options = { ModelOption.SEED: 123, ModelOption.TEMPERATURE: 0.5, - ModelOption.THINKING: True, ModelOption.MAX_NEW_TOKENS: 100, - "reasoning_effort": True, + + # Ollama thinking controls currently broken on Granite; see + # https://github.com/ollama/ollama/issues/10983 + # TODO: Re-enable when this upstream bug gets fixed. + #ModelOption.THINKING: True, + #"reasoning_effort": True, "homer_simpson": "option should be kicked out", } From 42268d7e9771412e03dcff2c91e755ad2e0eeabb Mon Sep 17 00:00:00 2001 From: jakelorocco <59755218+jakelorocco@users.noreply.github.com> Date: Sat, 15 Nov 2025 08:35:00 -0500 Subject: [PATCH 37/38] fix: make huggingface adapter test qualitative --- test/backends/test_huggingface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/backends/test_huggingface.py b/test/backends/test_huggingface.py index f0854066d..79434097c 100644 --- a/test/backends/test_huggingface.py +++ b/test/backends/test_huggingface.py @@ -40,7 +40,7 @@ def session(backend): yield session session.reset() - +@pytest.mark.qualitative def test_adapters(backend): assert len(backend._added_adapters.items()) > 0 From 25196e194c0044bb129a5cc3967ee97be4066059 Mon Sep 17 00:00:00 2001 From: Jake LoRocco Date: Mon, 17 Nov 2025 09:35:23 -0500 Subject: [PATCH 38/38] fix: issues with tests after refactor --- docs/examples/intrinsics/intrinsics.py | 32 +++++++++++++++----------- mellea/backends/huggingface.py | 3 ++- mellea/backends/openai.py | 3 ++- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/docs/examples/intrinsics/intrinsics.py b/docs/examples/intrinsics/intrinsics.py index 32c256aea..1392c5518 100644 --- a/docs/examples/intrinsics/intrinsics.py +++ b/docs/examples/intrinsics/intrinsics.py @@ -1,26 +1,32 @@ +from mellea.backends.huggingface import LocalHFBackend from mellea.backends.openai import OpenAIBackend, _ServerType from mellea.backends.adapters.adapter import AdapterType, GraniteCommonAdapter from mellea.stdlib.base import ChatContext, ModelOutputThunk from mellea.stdlib.chat import Message -import mellea.stdlib.funcs as mfuncs +import mellea.stdlib.functional as mfuncs from mellea.stdlib.intrinsics.intrinsic import Intrinsic -from mellea.stdlib.requirement import REQUIREMENT_REPO_ID -# Create the backend. Assumes a locally running VLLM server. -backend = OpenAIBackend( - model_id="ibm-granite/granite-3.3-8b-instruct", - base_url="http://0.0.0.0:8000/v1", - api_key="EMPTY", -) +# This is an example for how you would directly use intrinsics. See `mellea/stdlib/intrinsics/rag.py` +# for helper functions. + +# Create the backend. Example for a VLLM Server. Commented out in favor of the hugging face code for now. +# # Assumes a locally running VLLM server. +# backend = OpenAIBackend( +# model_id="ibm-granite/granite-3.3-8b-instruct", +# base_url="http://0.0.0.0:8000/v1", +# api_key="EMPTY", +# ) + +# # If using a remote VLLM server, utilize the `test/backends/test_openai_vllm/serve.sh` +# # script with `export VLLM_DOWNLOAD_RAG_INTRINSICS=True`. This will download the granite_common +# # adapters on the server. +# backend._server_type = _ServerType.REMOTE_VLLM -# If using a remote VLLM server, utilize the `test/backends/test_openai_vllm/serve.sh` -# script with `export VLLM_DOWNLOAD_RAG_INTRINSICS=True`. This will download the granite_common -# adapters on the server. -backend._server_type = _ServerType.REMOTE_VLLM +backend = LocalHFBackend(model_id="ibm-granite/granite-3.3-8b-instruct") # Create the Adapter. GraniteCommonAdapter's default to ALORAs. req_adapter = GraniteCommonAdapter( - REQUIREMENT_REPO_ID, "requirement_check", base_model_name=backend.base_model_name + "requirement_check", base_model_name=backend.base_model_name ) # Add the adapter to the backend. diff --git a/mellea/backends/huggingface.py b/mellea/backends/huggingface.py index 286dac024..1d82aeb07 100644 --- a/mellea/backends/huggingface.py +++ b/mellea/backends/huggingface.py @@ -218,10 +218,11 @@ def generate_from_context( ) if alora_req_adapter is None: # Log a warning if using an AloraRequirement but no adapter fit. - if reroute_to_alora: + if reroute_to_alora and isinstance(action, ALoraRequirement): FancyLogger.get_logger().warning( f"attempted to use an AloraRequirement but backend {self} doesn't have the specified adapter added {adapter_name}; defaulting to regular generation" ) + reroute_to_alora = False if issubclass(type(action), LLMaJRequirement): reroute_to_alora = False diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index f7109d489..d28766e96 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -338,10 +338,11 @@ def generate_from_chat_context( ) if alora_req_adapter is None: # Log a warning if using an AloraRequirement but no adapter fit. - if reroute_to_alora: + if reroute_to_alora and isinstance(action, ALoraRequirement): FancyLogger.get_logger().warning( f"attempted to use an AloraRequirement but backend {self} doesn't have the specified adapter added {adapter_name}; defaulting to regular generation" ) + reroute_to_alora = False if issubclass(type(action), LLMaJRequirement): reroute_to_alora = False