diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 2e490ade3..79881bbc5 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.10.79" +version = "2.10.80" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/src/uipath/eval/models/evaluation_set.py b/packages/uipath/src/uipath/eval/models/evaluation_set.py index 22e6ce244..c80da8e14 100644 --- a/packages/uipath/src/uipath/eval/models/evaluation_set.py +++ b/packages/uipath/src/uipath/eval/models/evaluation_set.py @@ -1,8 +1,9 @@ """Evaluation set models.""" +import re from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator from pydantic.alias_generators import to_camel from ..mocks._types import ( @@ -15,6 +16,21 @@ LegacyConversationalEvalOutput, ) +_GUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) + + +def normalize_eval_id(value: str) -> str: + """Canonicalize a GUID id to lowercase; leave non-GUID ids unchanged. + + GUIDs are case-insensitive, but downstream correlation (selection, + span/cache keying) compares ids as plain strings, so a mixed-case id + must be normalized at ingestion to stay matchable. + """ + return value.lower() if isinstance(value, str) and _GUID_RE.match(value) else value + class EvaluatorReference(BaseModel): """Reference to an evaluator with optional weight. @@ -96,6 +112,12 @@ class EvaluationItem(BaseModel): alias="inputMockingStrategy", ) + @field_validator("id") + @classmethod + def _normalize_id(cls, value: str) -> str: + """Normalize GUID ids to canonical lowercase.""" + return normalize_eval_id(value) + class LegacyEvaluationItem(BaseModel): """Individual evaluation item within an evaluation set.""" @@ -130,6 +152,12 @@ class LegacyEvaluationItem(BaseModel): default=None, alias="conversationalExpectedOutput" ) + @field_validator("id") + @classmethod + def _normalize_id(cls, value: str) -> str: + """Normalize GUID ids to canonical lowercase.""" + return normalize_eval_id(value) + class EvaluationSet(BaseModel): """Complete evaluation set model.""" @@ -153,7 +181,7 @@ class EvaluationSet(BaseModel): def extract_selected_evals(self, eval_ids) -> None: """Filter evaluations to only include those with specified IDs.""" selected_evals: list[EvaluationItem] = [] - remaining_ids = set(eval_ids) + remaining_ids = {normalize_eval_id(eval_id) for eval_id in eval_ids} for evaluation in self.evaluations: if evaluation.id in remaining_ids: selected_evals.append(evaluation) @@ -187,7 +215,7 @@ class LegacyEvaluationSet(BaseModel): def extract_selected_evals(self, eval_ids) -> None: """Filter evaluations to only include those with specified IDs.""" selected_evals: list[LegacyEvaluationItem] = [] - remaining_ids = set(eval_ids) + remaining_ids = {normalize_eval_id(eval_id) for eval_id in eval_ids} for evaluation in self.evaluations: if evaluation.id in remaining_ids: selected_evals.append(evaluation) diff --git a/packages/uipath/tests/cli/eval/test_eval_id_casing.py b/packages/uipath/tests/cli/eval/test_eval_id_casing.py new file mode 100644 index 000000000..e14a88a68 --- /dev/null +++ b/packages/uipath/tests/cli/eval/test_eval_id_casing.py @@ -0,0 +1,69 @@ +"""Tests for case-insensitive eval id handling (PC-4688). + +Eval sets exported by some tools emit uppercase GUID ids. The backend +canonicalizes GUIDs to lowercase, so any case-sensitive correlation on the +runtime side (selection, span/cache keying) silently fails to match. These +tests pin the fix: GUID ids are normalized to lowercase at ingestion and +selection is casing-agnostic. +""" + +from typing import Any + +from uipath.eval.models.evaluation_set import ( + EvaluationItem, + EvaluationSet, + LegacyEvaluationItem, +) + +UPPER_GUID = "B063907C-76AB-4B0A-88A3-EC0FB40698B8" +LOWER_GUID = "b063907c-76ab-4b0a-88a3-ec0fb40698b8" + + +def _make_item(eval_id: str) -> dict[str, Any]: + return { + "id": eval_id, + "name": "item", + "inputs": {"x": 1}, + "evaluationCriterias": {}, + } + + +def test_evaluation_item_normalizes_uppercase_guid_id(): + """An uppercase GUID id is stored in canonical lowercase form.""" + item = EvaluationItem.model_validate(_make_item(UPPER_GUID)) + assert item.id == LOWER_GUID + + +def test_legacy_evaluation_item_normalizes_uppercase_guid_id(): + """LegacyEvaluationItem also normalizes uppercase GUID ids.""" + item = LegacyEvaluationItem.model_validate( + { + "id": UPPER_GUID, + "name": "item", + "inputs": {"x": 1}, + "expectedOutput": {}, + "evalSetId": "set-1", + "createdAt": "2025-01-01T00:00:00.000Z", + "updatedAt": "2025-01-01T00:00:00.000Z", + } + ) + assert item.id == LOWER_GUID + + +def test_non_guid_id_is_left_unchanged(): + """Non-GUID ids (e.g. slugs) keep their original value and casing.""" + item = EvaluationItem.model_validate(_make_item("Test-Eval-1")) + assert item.id == "Test-Eval-1" + + +def test_extract_selected_evals_matches_regardless_of_caller_casing(): + """Selecting by an uppercase GUID matches a normalized stored id.""" + eval_set = EvaluationSet.model_validate( + { + "id": "set-1", + "name": "set", + "evaluations": [_make_item(LOWER_GUID), _make_item("other-id")], + } + ) + eval_set.extract_selected_evals([UPPER_GUID]) + assert [e.id for e in eval_set.evaluations] == [LOWER_GUID] diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 476c94d05..f78dd4cbe 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2552,7 +2552,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.10.79" +version = "2.10.80" source = { editable = "." } dependencies = [ { name = "applicationinsights" },