chore: Update expected test outputs to reflect upstream config changes

mellea/stdlib/components/intrinsic/rag.py

@@ -30,14 +30,14 @@ def check_answerability(
             for answerability checks.
 
     Returns:
-        Answerability score as a floating-point value from 0 to 1.
+        A string value of either "answerable" or "unanswerable"
     """
     result_json = call_intrinsic(
         "answerability",
         context.add(Message("user", question, documents=list(documents))),
         backend,
     )
-    return result_json["answerability_likelihood"]
+    return result_json["answerability"]
 
 
 def rewrite_question(
@@ -146,7 +146,10 @@ def check_context_relevance(
             intrinsic.
 
     Returns:
-        Context relevance score as a floating-point value from 0 to 1.
+        Context relevance judgement as one of the following strings:
+        - "relevant"
+        - "irrelevant"
+        - "partially relevant"
     """
     result_json = call_intrinsic(
         "context_relevance",
@@ -180,7 +183,7 @@ def flag_hallucinated_content(
 
     Returns:
         List of records with the following fields: ``response_begin``,
-        ``response_end``, ``response_text``, ``faithfulness_likelihood``,
+        ``response_end``, ``response_text``, ``faithfulness``,
         ``explanation``.
     """
     result_json = call_intrinsic(

test/formatters/granite/test_intrinsics_formatters.py

@@ -29,7 +29,6 @@
 )
 from mellea.formatters.granite.base import util as base_util
 from mellea.formatters.granite.intrinsics import json_util, util as intrinsics_util
-from test.predicates import require_gpu
 
 
 def _read_file(name):
@@ -434,6 +433,9 @@ def test_canned_input(yaml_json_combo_no_alora):
     assert after_json == expected_json
 
 
+# PLEASE DO NOT REMOVE THIS DECORATOR.
+# This decorator activates the network blocking features of the pytest-recording plugin.
+@pytest.mark.block_network
 def test_openai_compat(yaml_json_combo_no_alora):
     """
     Verify that the dataclasses for intrinsics chat completions can be directly passed
@@ -573,12 +575,12 @@ def _round_floats(json_data, num_digits: int = 2):
     return result
 
 
+# THIS TEST DOES NOT REQUIRE A GPU.
+# Please do not mark it with @require_gpu.
+# THIS TEST USES gh_run TO SKIP EXPENSIVE WORK WHEN RUNNING ON CI.
+# Please do not mark this test with @pytest.mark.skipif(os.environ.get("CICD"))
 @pytest.mark.huggingface
 @pytest.mark.e2e
-@require_gpu(min_vram_gb=12)
-@pytest.mark.skipif(
-    int(os.environ.get("CICD", 0)) == 1, reason="Skipping HuggingFace tests in CI"
-)
 def test_run_transformers(yaml_json_combo_with_model, gh_run):
     """
     Run the target model end-to-end on transformers.
@@ -610,30 +612,30 @@ def test_run_transformers(yaml_json_combo_with_model, gh_run):
         pytest.xfail("Downloads fail on CI server because repo is private")
 
     # Load IO config YAML for this model
-    io_yaml_path = lora_dir / "io.yaml"
-    if not os.path.exists(io_yaml_path):
-        # Use local files until proper configs are up on Hugging Face
-        io_yaml_path = cfg.yaml_file
+    io_yaml_path = cfg.yaml_file if cfg.yaml_file else lora_dir / "io.yaml"
     rewriter = IntrinsicsRewriter(config_file=io_yaml_path)
     result_processor = IntrinsicsResultProcessor(config_file=io_yaml_path)
 
     # Prepare inputs for inference
     transformed_input = rewriter.transform(model_input, **transform_kwargs)
 
-    if gh_run:
+    if gh_run == 1:
         pytest.xfail(
             "Skipping end-to-end model evaluation for this test case because it takes "
             "more than 5 seconds. "
-            "Mellea's CI fails the entire run without an error message if all 500+ "
+            "Mellea's CI fails the entire run without an error message if all 1900+ "
             "tests combined take more than 15 minutes to complete. "
-            "That works out to 1.8 seconds per test. "
+            "That works out to 0.5 seconds per test. "
             "Any test that takes more than 5 seconds needs to disable or shortcut "
             "itself during CI, or all of Mellea's development infrastructure will "
             "grind to a halt."
         )
 
     # Run the model using Hugging Face APIs
     model, tokenizer = base_util.load_transformers_lora(lora_dir)
+    if torch.cuda.is_available():  # Use GPU if available
+        model.cuda()
+
     generate_input, other_input = (
         base_util.chat_completion_request_to_transformers_inputs(
             transformed_input.model_dump(), tokenizer, model

test/formatters/granite/testdata/input_yaml/answerability.yaml

@@ -7,16 +7,10 @@ response_format: |
     "enum": ["answerable", "unanswerable"]
   }
 transformations:
-  # Convert categorical answer to continuous value by decoding logprobs
-  - type: likelihood
-    categories_to_values:
-      "answerable": 1.0
-      "unanswerable": 0.0
-    input_path: []
   # Convert scalar value to a record for consistency with other intrinsics
   - type: nest
     input_path: []
-    field_name: "answerability_likelihood"
+    field_name: "answerability"
 instruction: ~
 parameters:
   # "unanswerable" can be 6 tokens at high temperatures

test/formatters/granite/testdata/test_canned_input/answerability_answerable.json

@@ -26,7 +26,5 @@
       }
     }
   },
-  "max_completion_tokens": 6,
-  "logprobs": true,
-  "top_logprobs": 10
+  "max_completion_tokens": 6
 }

test/formatters/granite/testdata/test_canned_input/answerability_extra_params.json

@@ -19,7 +19,5 @@
   },
   "frequency_penalty": 0.1,
   "n": 5,
-  "max_completion_tokens": 6,
-  "logprobs": true,
-  "top_logprobs": 10
+  "max_completion_tokens": 6
 }

test/formatters/granite/testdata/test_canned_input/answerability_simple.json

@@ -16,7 +16,5 @@
       }
     }
   },
-  "max_completion_tokens": 6,
-  "logprobs": true,
-  "top_logprobs": 10
+  "max_completion_tokens": 6
 }

test/formatters/granite/testdata/test_canned_input/answerability_unanswerable.json

@@ -30,7 +30,5 @@
       }
     }
   },
-  "max_completion_tokens": 6,
-  "logprobs": true,
-  "top_logprobs": 10
+  "max_completion_tokens": 6
 }

test/formatters/granite/testdata/test_canned_input/context_relevance.json

@@ -30,7 +30,5 @@
         ]
       }
     }
-  },
-  "logprobs": true,
-  "top_logprobs": 10
+  }
 }

test/formatters/granite/testdata/test_canned_input/gpt_oss_answerability.json

@@ -24,7 +24,5 @@
       }
     }
   },
-  "max_completion_tokens": 25,
-  "logprobs": true,
-  "top_logprobs": 10
+  "max_completion_tokens": 25
 }

test/formatters/granite/testdata/test_canned_input/gpt_oss_hallucination_detection.json

@@ -64,7 +64,5 @@
     }
   },
   "temperature": 0.0,
-  "max_completion_tokens": 4096,
-  "logprobs": true,
-  "top_logprobs": 10
+  "max_completion_tokens": 4096
 }

test/formatters/granite/testdata/test_canned_input/hallucination_detection.json

@@ -66,7 +66,5 @@
     }
   },
   "temperature": 0.0,
-  "max_completion_tokens": 4096,
-  "logprobs": true,
-  "top_logprobs": 10
+  "max_completion_tokens": 4096
 }

test/formatters/granite/testdata/test_canned_output/expected_result/answerability_answerable.json

@@ -3,27 +3,9 @@
         {
             "index": 0,
             "message": {
-                "content": "{\"answerability_likelihood\": 0.9991958643582448}",
-                "role": "assistant",
-                "tool_calls": [],
-                "reasoning_content": null
-            },
-            "finish_reason": "stop"
+                "content": "{\"answerability\": \"answerable\"}",
+                "role": "assistant"
+            }
         }
-    ],
-    "prompt_logprobs": null,
-    "id": "chatcmpl-4265b3bad97e4177b7c0dc15b0fe2611",
-    "created": 1756335561,
-    "model": "answerability",
-    "object": "chat.completion",
-    "service_tier": null,
-    "system_fingerprint": null,
-    "usage": {
-        "completion_tokens": 5,
-        "prompt_tokens": 144,
-        "total_tokens": 149,
-        "completion_tokens_details": null,
-        "prompt_tokens_details": null
-    },
-    "kv_transfer_params": null
-}
+    ]
+}

test/formatters/granite/testdata/test_canned_output/expected_result/answerability_simple.json

@@ -3,7 +3,7 @@
         {
             "index": 0,
             "message": {
-                "content": "{\"answerability_likelihood\": 0.09729512381348375}",
+                "content": "{\"answerability\": \"unanswerable\"}",
                 "role": "assistant"
             }
         }

test/formatters/granite/testdata/test_canned_output/expected_result/answerability_unanswerable.json

@@ -3,27 +3,9 @@
         {
             "index": 0,
             "message": {
-                "content": "{\"answerability_likelihood\": 0.00020351838117683546}",
-                "role": "assistant",
-                "tool_calls": [],
-                "reasoning_content": null
-            },
-            "finish_reason": "length"
+                "content": "{\"answerability\": \"unanswerable\"}",
+                "role": "assistant"
+            }
         }
-    ],
-    "prompt_logprobs": null,
-    "id": "chatcmpl-1679360dff7f4e358cfeb205ee11e692",
-    "created": 1756336771,
-    "model": "answerability",
-    "object": "chat.completion",
-    "service_tier": null,
-    "system_fingerprint": null,
-    "usage": {
-        "completion_tokens": 6,
-        "prompt_tokens": 151,
-        "total_tokens": 157,
-        "completion_tokens_details": null,
-        "prompt_tokens_details": null
-    },
-    "kv_transfer_params": null
+    ]
 }

test/formatters/granite/testdata/test_canned_output/expected_result/citations.json

@@ -27,4 +27,4 @@
         "prompt_tokens_details": null
     },
     "kv_transfer_params": null
-}
+}

test/formatters/granite/testdata/test_canned_output/expected_result/context_relevance.json

@@ -3,27 +3,9 @@
         {
             "index": 0,
             "message": {
-                "content": "{\"context_relevance\": 0.47131842717316647}",
-                "role": "assistant",
-                "tool_calls": [],
-                "reasoning_content": null
-            },
-            "finish_reason": "stop"
+                "content": "{\"context_relevance\": \"irrelevant\"}",
+                "role": "assistant"
+            }
         }
-    ],
-    "prompt_logprobs": null,
-    "id": "chatcmpl-d2739b219b984788b765b807d0696697",
-    "created": 1756859041,
-    "model": "context_relevance",
-    "object": "chat.completion",
-    "service_tier": null,
-    "system_fingerprint": null,
-    "usage": {
-        "completion_tokens": 13,
-        "prompt_tokens": 252,
-        "total_tokens": 265,
-        "completion_tokens_details": null,
-        "prompt_tokens_details": null
-    },
-    "kv_transfer_params": null
+    ]
 }

test/formatters/granite/testdata/test_canned_output/expected_result/gpt_oss_answerability.json

@@ -3,11 +3,11 @@
         {
             "index": 0,
             "message": {
-                "content": "{\"answerability_likelihood\": 0.9669141293813437}",
+                "content": "{\"answerability\": \"answerable\"}",
                 "role": "assistant",
                 "tool_calls": []
             },
             "finish_reason": "stop"
         }
     ]
-}
+}

test/formatters/granite/testdata/test_canned_output/expected_result/gpt_oss_hallucination_detection.json

@@ -3,11 +3,11 @@
         {
             "index": 0,
             "message": {
-                "content": "[{\"response_begin\": 0, \"response_end\": 31, \"response_text\": \"Purple bumble fish are yellow. \", \"faithfulness_likelihood\": 0.9395809657567371, \"explanation\": \"This sentence makes a factual claim about the color of purple bumble fish. The provided context states: 'The only type of fish that is yellow is the purple bumble fish.' This directly supports the claim in the sentence.\"}, {\"response_begin\": 31, \"response_end\": 65, \"response_text\": \"Green bumble fish are also yellow.\", \"faithfulness_likelihood\": 0.05383118091859103, \"explanation\": \"This sentence makes a factual claim about the color of green bumble fish. However, the provided context does not mention green bumble fish at all, so there is no evidence to support this claim.\"}]",
+                "content": "[{\"response_begin\": 0, \"response_end\": 31, \"response_text\": \"Purple bumble fish are yellow. \", \"faithfulness\": \"faithful\", \"explanation\": \"This sentence makes a factual claim about the color of purple bumble fish. The provided context states: 'The only type of fish that is yellow is the purple bumble fish.' This directly supports the claim in the sentence.\"}, {\"response_begin\": 31, \"response_end\": 65, \"response_text\": \"Green bumble fish are also yellow.\", \"faithfulness\": \"unfaithful\", \"explanation\": \"This sentence makes a factual claim about the color of green bumble fish. However, the provided context does not mention green bumble fish at all, so there is no evidence to support this claim.\"}]",
                 "role": "assistant",
                 "tool_calls": []
             },
             "finish_reason": "stop"
         }
     ]
-}
+}

test/formatters/granite/testdata/test_canned_output/expected_result/hallucination_detection.json

@@ -3,27 +3,9 @@
         {
             "index": 0,
             "message": {
-                "content": "[{\"response_begin\": 0, \"response_end\": 31, \"response_text\": \"Purple bumble fish are yellow. \", \"faithfulness_likelihood\": 0.07605912763921696, \"explanation\": \"This sentence makes a factual claim about the color of purple bumble fish. However, the provided context states: 'The only type of fish that is yellow is the purple bumble fish.' This implies that only one fish, the purple bumble fish, is yellow, not 'Green bumble fish are also yellow.'\"}, {\"response_begin\": 31, \"response_end\": 65, \"response_text\": \"Green bumble fish are also yellow.\", \"faithfulness_likelihood\": 0.0003170815425561981, \"explanation\": \"This sentence makes a factual claim about the color of green bumble fish. However, there is no information in the provided context about green bumble fish. The only fish mentioned is the purple bumble fish.\"}]",
-                "role": "assistant",
-                "tool_calls": [],
-                "reasoning_content": null
-            },
-            "finish_reason": "stop"
+                "content": "[{\"response_begin\": 0, \"response_end\": 31, \"response_text\": \"Purple bumble fish are yellow. \", \"faithfulness\": \"faithful\", \"explanation\": \"This sentence makes a factual claim about the color of purple bumble fish. The document states 'The only type of fish that is yellow is the purple bumble fish.' This directly supports the claim in the sentence.\"}, {\"response_begin\": 31, \"response_end\": 65, \"response_text\": \"Green bumble fish are also yellow.\", \"faithfulness\": \"unfaithful\", \"explanation\": \"This sentence makes a factual claim about the color of green bumble fish. However, the document does not mention green bumble fish at all. Therefore, this claim cannot be verified from the provided context.\"}]",
+                "role": "assistant"
+            }
         }
-    ],
-    "prompt_logprobs": null,
-    "id": "chatcmpl-94996468501f4912841a858d09e2c895",
-    "created": 1758241060,
-    "model": "hallucination_detection",
-    "object": "chat.completion",
-    "service_tier": null,
-    "system_fingerprint": null,
-    "usage": {
-        "completion_tokens": 153,
-        "prompt_tokens": 275,
-        "total_tokens": 428,
-        "completion_tokens_details": null,
-        "prompt_tokens_details": null
-    },
-    "kv_transfer_params": null
-}
+    ]
+}