From ce69936f00a0de470a43025472d4b7fcc4e5ebb5 Mon Sep 17 00:00:00 2001 From: Lucas Pickup Date: Thu, 1 Feb 2024 20:48:40 -0800 Subject: [PATCH 1/3] Set `use_rcts` in `build_index` helper used by local index build. Recursive Text Splitter is enabled by default for cloud index builds and effects how Markdown files are chunked. It preferences chunk length being close to the given `chunk_size` over splitting by heading sections. --- .../azure/ai/generative/index/_build_mlindex.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sdk/ai/azure-ai-generative/azure/ai/generative/index/_build_mlindex.py b/sdk/ai/azure-ai-generative/azure/ai/generative/index/_build_mlindex.py index 1db912ebbf69..60fb5e0306dd 100644 --- a/sdk/ai/azure-ai-generative/azure/ai/generative/index/_build_mlindex.py +++ b/sdk/ai/azure-ai-generative/azure/ai/generative/index/_build_mlindex.py @@ -4,14 +4,14 @@ from pathlib import Path from typing import Dict, Optional, Union -from packaging import version import yaml # type: ignore[import] +from packaging import version +from azure.ai.resources._utils._open_ai_utils import build_open_ai_protocol from azure.ai.resources.entities.mlindex import Index -from azure.ai.resources.operations._index_data_source import ACSSource, LocalSource from azure.ai.resources.operations._acs_output_config import ACSOutputConfig -from azure.ai.resources._utils._open_ai_utils import build_open_ai_protocol +from azure.ai.resources.operations._index_data_source import ACSSource, LocalSource def build_index( @@ -60,6 +60,7 @@ def build_index( splitter_args= { 'chunk_size': chunk_size, 'chunk_overlap': chunk_overlap, + 'use_rcts': True } if max_sample_files is not None: splitter_args["max_sample_files"] = max_sample_files From d70f88b3d4734c58a1db5f97b2d30919b479c46e Mon Sep 17 00:00:00 2001 From: Lucas Pickup Date: Thu, 1 Feb 2024 20:51:57 -0800 Subject: [PATCH 2/3] Udpate changelog. --- sdk/ai/azure-ai-generative/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/ai/azure-ai-generative/CHANGELOG.md b/sdk/ai/azure-ai-generative/CHANGELOG.md index 2554bf02f6a8..95a740ede97a 100644 --- a/sdk/ai/azure-ai-generative/CHANGELOG.md +++ b/sdk/ai/azure-ai-generative/CHANGELOG.md @@ -34,6 +34,7 @@ - Switched evaluate() function to run on promptflow engine - Use openai v1 environment variable +- Recursive Text Splitter is now enabled by default for local index builds which mimics behavior of cloud index builds ## 1.0.0b3 (2024-01-30) From 2cbac3563a43122048689bcdbd78eb2388458bcb Mon Sep 17 00:00:00 2001 From: Lucas Pickup Date: Sun, 4 Feb 2024 22:29:29 -0800 Subject: [PATCH 3/3] Move changelog to latest unreleased version. --- sdk/ai/azure-ai-generative/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/ai/azure-ai-generative/CHANGELOG.md b/sdk/ai/azure-ai-generative/CHANGELOG.md index 95a740ede97a..81982c431cc5 100644 --- a/sdk/ai/azure-ai-generative/CHANGELOG.md +++ b/sdk/ai/azure-ai-generative/CHANGELOG.md @@ -10,6 +10,8 @@ ### Other Changes +- Recursive Text Splitter is now enabled by default for local index builds which mimics behavior of cloud index builds + ## 1.0.0b5 (2024-02-02) ### Features Added @@ -34,7 +36,6 @@ - Switched evaluate() function to run on promptflow engine - Use openai v1 environment variable -- Recursive Text Splitter is now enabled by default for local index builds which mimics behavior of cloud index builds ## 1.0.0b3 (2024-01-30)