From 480200857dde57b936991318d77a68061e9bbad7 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Wed, 16 Apr 2025 16:06:34 -0700 Subject: [PATCH 1/2] Allow for alphanumeric in column mapping --- sdk/evaluation/azure-ai-evaluation/CHANGELOG.md | 7 +++++++ .../azure/ai/evaluation/_evaluate/_evaluate.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md index ce9d3f5ee0bb..0cec017a26d7 100644 --- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md +++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md @@ -7,6 +7,13 @@ ### Breaking Changes ### Bugs Fixed +- Fixed error in `evaluate` where data fields could not contain numeric characters. Previously, a data file with schema: + ``` + "query1": "some query", "response: "some response" + ``` + throws error when passed into `evaluator_config` as `{"evaluator_name": {"column_mapping": {"query": "${data.query1}", "response": "${data.response}"}},}`. + Now, users may import data containing fields with numeric characters. + ### Other Changes diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py index 096ab85551be..99f9537a56cf 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py @@ -573,7 +573,7 @@ def _process_column_mappings( processed_config: Dict[str, Dict[str, str]] = {} - expected_references = re.compile(r"^\$\{(target|data)\.[a-zA-Z_]+\}$") + expected_references = re.compile(r"^\$\{(target|data)\.[a-zA-Z0-9_]+\}$") if column_mapping: for evaluator, mapping_config in column_mapping.items(): From 259b151caf5b0f6d2fcc265d05868e79d03e55f2 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Sun, 20 Apr 2025 12:14:35 -0700 Subject: [PATCH 2/2] add unit test --- .../evaluate_test_data_alphanumeric.jsonl | 3 ++ .../tests/unittests/test_evaluate.py | 41 +++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluate_test_data_alphanumeric.jsonl diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluate_test_data_alphanumeric.jsonl b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluate_test_data_alphanumeric.jsonl new file mode 100644 index 000000000000..4121bb0a0354 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/evaluate_test_data_alphanumeric.jsonl @@ -0,0 +1,3 @@ +{"query456":"How do you create a run?","context789":"AML API only","response123":"To create a run using the Azure Machine Learning API, you first need to create an Experiment. Once you have an experiment, you can create a Run object that is associated with that experiment. Here is some Python code that demonstrates this process:\n\n```\nfrom azureml.core import Experiment, Run\nfrom azureml.core.workspace import Workspace\n\n# Define workspace and experiment\nws = Workspace.from_config()\nexp = Experiment(workspace=ws, name='my_experiment')\n\n# Create a new run\nrun = exp.start_logging()\n```\n\nIn this code, the `from_config()` method reads the configuration file that you created when you set up your Azure Machine Learning workspace. The `Experiment` constructor creates an Experiment object that is associated with your workspace, and the `start_logging()` method creates a new Run object that is associated with the Experiment. Now you can use the `run` object to log metrics, upload files, and track other information related to your machine learning experiment.", "ground_truth":"Paris is the capital of France."} +{"query456":"How do you log a model?","context789":"Logging can be done using any OSS Sdk","response123":"There are a few ways to log models in Azure Machine Learning. \n\nOne way is to use the `register_model()` method of the `Run` object. The `register_model()` method logs a model file in the Azure Machine Learning service workspace and makes it available for deployment. Here's an example:\n\n```python\nfrom azureml.core import Model\n\nmodel_path = '.\/outputs\/my_model.pkl'\nmodel = Model.register(workspace=ws, model_path=model_path, model_name='my_model')\n```\n\nThis code registers the model file located at `model_path` to the Azure Machine Learning service workspace with the name `my_model`. \n\nAnother way to log a model is to save it as an output of a `Run`. If your model generation code is part of a script or Jupyter notebook that runs as an Azure Machine Learning experiment, you can save the model file as an output of the `Run` object. Here's an example:\n\n```python\nfrom sklearn.linear_model import LogisticRegression\nfrom azureml.core.run import Run\n\n# Initialize a run object\nrun = Run.get_context789()\n\n# Train your model\nX_train, y_train = ...\nclf = LogisticRegression().fit(X_train, y_train)\n\n# Save the model to the Run object's outputs directory\nmodel_path = 'outputs\/model.pkl'\njoblib.dump(value=clf, filename=model_path)\n\n# Log the model as a run artifact\nrun.upload_file(name=model_path, path_or_stream=model_path)\n```\n\nIn this code, `Run.get_context789()` retrieves the current run context789 object, which you can use to track metadata and metrics for the run. After training your model, you can use `joblib.dump()` to save the model to a file, and then log the file as an artifact of the run using `run.upload_file()`.","ground_truth":"Paris is the capital of France."} +{"query456":"What is the capital of France?","context789":"France is in Europe","response123":"Paris is the capital of France.", "ground_truth":"Paris is the capital of France."} diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 2ca4979b9004..fe0da7a08ee7 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -70,6 +70,9 @@ def evaluate_test_data_jsonl_file(): def evaluate_test_data_conversion_jsonl_file(): return _get_file("evaluate_test_data_conversation.jsonl") +@pytest.fixture +def evaluate_test_data_alphanumeric(): + return _get_file("evaluate_test_data_alphanumeric.jsonl") @pytest.fixture def questions_file(): @@ -414,10 +417,42 @@ def test_evaluate_invalid_column_mapping(self, mock_model_config, evaluate_test_ }, ) - assert ( - "Unexpected references detected in 'column_mapping'. Ensure only ${target.} and ${data.} are used." - in exc_info.value.args[0] + assert ( + "Unexpected references detected in 'column_mapping'. Ensure only ${target.} and ${data.} are used." + in exc_info.value.args[0] + ) + + def test_evaluate_valid_column_mapping_with_numeric_chars(self, mock_model_config, evaluate_test_data_alphanumeric): + # Valid column mappings that include numeric characters + # This test validates the fix for the regex pattern that now accepts numeric characters + # Previous regex was `re.compile(r"^\$\{(target|data)\.[a-zA-Z_]+\}$")` + # New regex is `re.compile(r"^\$\{(target|data)\.[a-zA-Z0-9_]+\}$")` + + column_mappings_with_numbers = { + "response": "${data.response123}", + "query": "${data.query456}", + "context": "${data.context789}" + } # This should not raise an exception with the updated regex for column mapping format validation + # The test passes if no exception about "Unexpected references" is raised + result = evaluate( + data=evaluate_test_data_alphanumeric, + evaluators={"g": GroundednessEvaluator(model_config=mock_model_config)}, + evaluator_config={ + "g": { + "column_mapping": column_mappings_with_numbers, + } + }, + fail_on_evaluator_errors=False ) + + # Verify that the test completed without errors related to column mapping format + # The test data has the fields with numeric characters, so it should work correctly + assert result is not None + # Verify we're getting data from the numerically-named fields + row_result_df = pd.DataFrame(result["rows"]) + assert "inputs.response123" in row_result_df.columns + assert "inputs.query456" in row_result_df.columns + assert "inputs.context789" in row_result_df.columns def test_renaming_column(self): """Test that the columns are renamed correctly."""