From b4294649fd4fa62de895297513c68ac1c838aab9 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Mon, 3 Nov 2025 11:32:22 -0700 Subject: [PATCH 01/21] add IndexRange and PartitionBlock --- src/data_designer/config/seed.py | 28 ++++++++++++++++++++++-- src/data_designer/essentials/__init__.py | 4 +++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/data_designer/config/seed.py b/src/data_designer/config/seed.py index 9f7480ebb..7f4e351b7 100644 --- a/src/data_designer/config/seed.py +++ b/src/data_designer/config/seed.py @@ -4,8 +4,9 @@ from abc import ABC from enum import Enum -from pydantic import field_validator - +from pydantic import field_validator, Field, model_validator +from typing_extensions import Self +from typing import Union, Optional from .base import ConfigBase from .datastore import DatastoreSettings from .utils.io_helpers import validate_dataset_file_path @@ -16,9 +17,32 @@ class SamplingStrategy(str, Enum): SHUFFLE = "shuffle" +class IndexRange(ConfigBase): + start: int = Field(..., ge=0) + end: int = Field(..., ge=1) + + @model_validator(mode="after") + def _validate_index_range(self) -> Self: + if self.start >= self.end: + raise ValueError("'start' index must be less than 'end' index") + return self + + +class PartitionBlock(ConfigBase): + partition_index: int = Field(..., default=0, ge=0) + num_partitions: int = Field(..., default=1, ge=1) + + @model_validator(mode="after") + def _validate_partition_block(self) -> Self: + if self.partition_index >= self.num_partitions: + raise ValueError("'partition_index' must be less than 'num_partitions'") + return self + + class SeedConfig(ConfigBase): dataset: str sampling_strategy: SamplingStrategy = SamplingStrategy.ORDERED + selection_strategy: Optional[Union[IndexRange, PartitionBlock]] = None class SeedDatasetReference(ABC, ConfigBase): diff --git a/src/data_designer/essentials/__init__.py b/src/data_designer/essentials/__init__.py index 50e21493b..597073427 100644 --- a/src/data_designer/essentials/__init__.py +++ b/src/data_designer/essentials/__init__.py @@ -47,7 +47,7 @@ UniformSamplerParams, UUIDSamplerParams, ) -from ..config.seed import DatastoreSeedDatasetReference, SamplingStrategy, SeedConfig +from ..config.seed import DatastoreSeedDatasetReference, SamplingStrategy, SeedConfig, IndexRange, PartitionBlock from ..config.utils.code_lang import CodeLang from ..config.utils.misc import can_run_data_designer_locally from ..config.validator_params import ( @@ -85,6 +85,7 @@ "DatetimeSamplerParams", "ExpressionColumnConfig", "GaussianSamplerParams", + "IndexRange", "ImageContext", "ImageFormat", "InferenceParameters", @@ -100,6 +101,7 @@ "ModalityContext", "ModalityDataType", "ModelConfig", + "PartitionBlock", "PersonSamplerParams", "PoissonSamplerParams", "RemoteValidatorParams", From 0d4b9a0fcd611ca31c664adb7fb2ca6096c19364 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Mon, 3 Nov 2025 11:33:49 -0700 Subject: [PATCH 02/21] make check-all-fix --- src/data_designer/config/seed.py | 5 +++-- src/data_designer/essentials/__init__.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/data_designer/config/seed.py b/src/data_designer/config/seed.py index 7f4e351b7..c01672963 100644 --- a/src/data_designer/config/seed.py +++ b/src/data_designer/config/seed.py @@ -3,10 +3,11 @@ from abc import ABC from enum import Enum +from typing import Optional, Union -from pydantic import field_validator, Field, model_validator +from pydantic import Field, field_validator, model_validator from typing_extensions import Self -from typing import Union, Optional + from .base import ConfigBase from .datastore import DatastoreSettings from .utils.io_helpers import validate_dataset_file_path diff --git a/src/data_designer/essentials/__init__.py b/src/data_designer/essentials/__init__.py index 597073427..47fc223dd 100644 --- a/src/data_designer/essentials/__init__.py +++ b/src/data_designer/essentials/__init__.py @@ -47,7 +47,7 @@ UniformSamplerParams, UUIDSamplerParams, ) -from ..config.seed import DatastoreSeedDatasetReference, SamplingStrategy, SeedConfig, IndexRange, PartitionBlock +from ..config.seed import DatastoreSeedDatasetReference, IndexRange, PartitionBlock, SamplingStrategy, SeedConfig from ..config.utils.code_lang import CodeLang from ..config.utils.misc import can_run_data_designer_locally from ..config.validator_params import ( From 57d500ba3cf9ea9011d1c396526d526135d72390 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Mon, 3 Nov 2025 16:09:31 -0700 Subject: [PATCH 03/21] add support for IndexRange and PartitionBlock --- src/data_designer/config/seed.py | 27 +++- .../generators/seed_dataset.py | 47 ++++++- .../engine/column_generators/utils/errors.py | 3 + tests/config/test_seed.py | 52 ++++++++ .../generators/test_seed_dataset.py | 123 +++++++++++++++++- 5 files changed, 240 insertions(+), 12 deletions(-) create mode 100644 tests/config/test_seed.py diff --git a/src/data_designer/config/seed.py b/src/data_designer/config/seed.py index c01672963..9554a8503 100644 --- a/src/data_designer/config/seed.py +++ b/src/data_designer/config/seed.py @@ -19,19 +19,23 @@ class SamplingStrategy(str, Enum): class IndexRange(ConfigBase): - start: int = Field(..., ge=0) - end: int = Field(..., ge=1) + start: int = Field(ge=0, description="The start index of the index range (inclusive)") + end: int = Field(ge=0, description="The end index of the index range (inclusive)") @model_validator(mode="after") def _validate_index_range(self) -> Self: - if self.start >= self.end: - raise ValueError("'start' index must be less than 'end' index") + if self.start > self.end: + raise ValueError("'start' index must be less than or equal to 'end' index") return self + @property + def size(self) -> int: + return self.end - self.start + 1 + class PartitionBlock(ConfigBase): - partition_index: int = Field(..., default=0, ge=0) - num_partitions: int = Field(..., default=1, ge=1) + partition_index: int = Field(default=0, ge=0, description="The index of the partition to sample from") + num_partitions: int = Field(default=1, ge=1, description="The total number of partitions in the dataset") @model_validator(mode="after") def _validate_partition_block(self) -> Self: @@ -39,6 +43,17 @@ def _validate_partition_block(self) -> Self: raise ValueError("'partition_index' must be less than 'num_partitions'") return self + def to_index_range(self, dataset_size: int) -> IndexRange: + partition_size = dataset_size // self.num_partitions + start = self.partition_index * partition_size + + # For the last partition, extend to the end of the dataset to include remainder rows + if self.partition_index == self.num_partitions - 1: + end = dataset_size - 1 + else: + end = ((self.partition_index + 1) * partition_size) - 1 + return IndexRange(start=start, end=end) + class SeedConfig(ConfigBase): dataset: str diff --git a/src/data_designer/engine/column_generators/generators/seed_dataset.py b/src/data_designer/engine/column_generators/generators/seed_dataset.py index ffec22be4..87066abcc 100644 --- a/src/data_designer/engine/column_generators/generators/seed_dataset.py +++ b/src/data_designer/engine/column_generators/generators/seed_dataset.py @@ -7,12 +7,13 @@ import duckdb import pandas as pd -from data_designer.config.seed import SamplingStrategy +from data_designer.config.seed import SamplingStrategy, IndexRange, PartitionBlock from data_designer.engine.column_generators.generators.base import ( FromScratchColumnGenerator, GenerationStrategy, GeneratorMetadata, ) +from data_designer.engine.column_generators.utils.errors import SeedDatasetError from data_designer.engine.dataset_builders.multi_column_configs import SeedDatasetMultiColumnConfig from data_designer.engine.processing.utils import concat_datasets from data_designer.engine.resources.resource_provider import ResourceType @@ -58,11 +59,50 @@ def _initialize(self) -> None: self._df_remaining = None self._dataset_uri = self.resource_provider.datastore.get_dataset_uri(self.config.dataset) self._seed_dataset_size = self.duckdb_conn.execute(f"SELECT COUNT(*) FROM '{self._dataset_uri}'").fetchone()[0] + self._index_range = self._resolve_index_range() + + def _validate_selection_strategy(self) -> None: + err_msg = None + if self.config.selection_strategy is not None: + if isinstance(self.config.selection_strategy, IndexRange) and self.config.selection_strategy.end >= self._seed_dataset_size: + err_msg = f"Selection strategy 'end' index {self.config.selection_strategy.end} is out of bounds for dataset size {self._seed_dataset_size}" + elif isinstance(self.config.selection_strategy, PartitionBlock) and self.config.selection_strategy.num_partitions > self._seed_dataset_size: + err_msg = f"Selection strategy 'num_partitions' {self.config.selection_strategy.num_partitions} is out of bounds for dataset size {self._seed_dataset_size}" + if err_msg is not None: + raise SeedDatasetError(err_msg) + + def _resolve_index_range(self) -> IndexRange | None: + self._validate_selection_strategy() + index_range = None + if self.config.selection_strategy is not None: + if isinstance(self.config.selection_strategy, IndexRange): + index_range = self.config.selection_strategy + elif isinstance(self.config.selection_strategy, PartitionBlock): + index_range = self.config.selection_strategy.to_index_range(self._seed_dataset_size) + return index_range def _reset_batch_reader(self, num_records: int) -> None: shuffle = self.config.sampling_strategy == SamplingStrategy.SHUFFLE shuffle_query = " ORDER BY RANDOM()" if shuffle else "" - self._batch_reader = self.duckdb_conn.query(f"SELECT * FROM '{self._dataset_uri}'{shuffle_query}").record_batch( + + if self._index_range is not None: + # Use subquery with row_number() window function to filter by index range + # IndexRange uses 0-based indexing [start, end] inclusive, row_number() is 1-based + # To convert 0-based index i to 1-based row_number: row_number = i + 1 + # For inclusive range [start, end], we want: row_number > start AND row_number <= end + 1 + # This gives us 1-based rows [start+1, end+1] which maps to 0-based indices [start, end] + read_query = f""" + SELECT * EXCLUDE (row_num) FROM ( + SELECT *, row_number() OVER () as row_num + FROM '{self._dataset_uri}' + ) sub + WHERE row_num > {self._index_range.start} AND row_num <= {self._index_range.end + 1} + {shuffle_query} + """ + else: + read_query = f"SELECT * FROM '{self._dataset_uri}'{shuffle_query}" + + self._batch_reader = self.duckdb_conn.query(read_query).record_batch( batch_size=num_records ) @@ -70,6 +110,9 @@ def _sample_records(self, num_records: int) -> pd.DataFrame: logger.info(f"🌱 Sampling {num_records} records from seed dataset") logger.info(f" |-- seed dataset size: {self._seed_dataset_size} records") logger.info(f" |-- sampling strategy: {self.config.sampling_strategy}") + if self._index_range is not None: + logger.info(f" |-- selection strategy: {self.config.selection_strategy.model_dump_json()}") + logger.info(f" |-- seed dataset size after selection: {self._index_range.size} records") df_batch = pd.DataFrame() df_sample = pd.DataFrame() if self._df_remaining is None else self._df_remaining diff --git a/src/data_designer/engine/column_generators/utils/errors.py b/src/data_designer/engine/column_generators/utils/errors.py index 7820406d9..f467d862e 100644 --- a/src/data_designer/engine/column_generators/utils/errors.py +++ b/src/data_designer/engine/column_generators/utils/errors.py @@ -8,3 +8,6 @@ class PromptTemplateRenderError(DataDesignerError): ... class ExpressionTemplateRenderError(DataDesignerError): ... + + +class SeedDatasetError(DataDesignerError): ... diff --git a/tests/config/test_seed.py b/tests/config/test_seed.py new file mode 100644 index 000000000..dddeda4ef --- /dev/null +++ b/tests/config/test_seed.py @@ -0,0 +1,52 @@ +import pytest + +from data_designer.config.seed import IndexRange, PartitionBlock + +def test_index_range_validation(): + with pytest.raises(ValueError, match="should be greater than or equal to 0"): + IndexRange(start=-1, end=10) + + with pytest.raises(ValueError, match="should be greater than or equal to 0"): + IndexRange(start=0, end=-1) + + with pytest.raises(ValueError, match="'start' index must be less than or equal to 'end' index"): + IndexRange(start=11, end=10) + + +def test_index_range_size(): + assert IndexRange(start=0, end=10).size == 11 + assert IndexRange(start=1, end=10).size == 10 + assert IndexRange(start=0, end=0).size == 1 + + +def test_partition_block_validation(): + with pytest.raises(ValueError, match="should be greater than or equal to 0"): + PartitionBlock(partition_index=-1, num_partitions=10) + + with pytest.raises(ValueError, match="should be greater than or equal to 1"): + PartitionBlock(partition_index=0, num_partitions=0) + + with pytest.raises(ValueError, match="'partition_index' must be less than 'num_partitions'"): + PartitionBlock(partition_index=10, num_partitions=10) + + +def test_partition_block_to_index_range(): + index_range = PartitionBlock(partition_index=0, num_partitions=10).to_index_range(101) + assert index_range.start == 0 + assert index_range.end == 9 + assert index_range.size == 10 + + index_range = PartitionBlock(partition_index=1, num_partitions=10).to_index_range(105) + assert index_range.start == 10 + assert index_range.end == 19 + assert index_range.size == 10 + + index_range = PartitionBlock(partition_index=2, num_partitions=10).to_index_range(105) + assert index_range.start == 20 + assert index_range.end == 29 + assert index_range.size == 10 + + index_range = PartitionBlock(partition_index=9, num_partitions=10).to_index_range(105) + assert index_range.start == 90 + assert index_range.end == 104 + assert index_range.size == 15 diff --git a/tests/engine/column_generators/generators/test_seed_dataset.py b/tests/engine/column_generators/generators/test_seed_dataset.py index cebb68ca0..3b43b7ef2 100644 --- a/tests/engine/column_generators/generators/test_seed_dataset.py +++ b/tests/engine/column_generators/generators/test_seed_dataset.py @@ -10,14 +10,15 @@ import pytest from data_designer.config.columns import SeedDatasetColumnConfig -from data_designer.config.seed import SamplingStrategy +from data_designer.config.seed import SamplingStrategy, IndexRange, PartitionBlock from data_designer.engine.column_generators.generators.base import GenerationStrategy from data_designer.engine.column_generators.generators.seed_dataset import ( MAX_ZERO_RECORD_RESPONSE_FACTOR, SeedDatasetColumnGenerator, ) from data_designer.engine.dataset_builders.multi_column_configs import SeedDatasetMultiColumnConfig -from data_designer.engine.resources.resource_provider import ResourceType +from data_designer.engine.resources.resource_provider import ResourceType, ResourceProvider +from data_designer.engine.column_generators.utils.errors import SeedDatasetError @pytest.fixture @@ -333,7 +334,11 @@ def test_seed_dataset_column_generator_sample_records_multiple_batches(stub_seed # ============================================================================ -def create_generator_with_real_file(file_path: str, stub_resource_provider) -> SeedDatasetColumnGenerator: +def create_generator_with_real_file( + file_path: str, + stub_resource_provider: ResourceProvider, + sampling_strategy: SamplingStrategy = SamplingStrategy.ORDERED, + selection_strategy: IndexRange | PartitionBlock | None = None) -> SeedDatasetColumnGenerator: """Helper function to create a generator with a real file and DuckDB connection.""" config = SeedDatasetMultiColumnConfig( columns=[ @@ -344,7 +349,8 @@ def create_generator_with_real_file(file_path: str, stub_resource_provider) -> S SeedDatasetColumnConfig(name="score"), ], dataset=f"test/{os.path.basename(file_path)}", - sampling_strategy=SamplingStrategy.ORDERED, + sampling_strategy=sampling_strategy, + selection_strategy=selection_strategy, ) # Create a real DuckDB connection (in-memory by default) @@ -605,3 +611,112 @@ def test_seed_dataset_generator_uses_real_duckdb_connection(fixture_name, stub_r # Verify the connection can execute count queries count_result = generator.duckdb_conn.execute(f"SELECT COUNT(*) FROM '{file_path}'").fetchone()[0] assert count_result == 10 + + +# ============================================================================ +# Tests for SeedConfig selection strategies +# ============================================================================ +@pytest.mark.parametrize( + "fixture_name", + [ + "seed_dataset_parquet", + "seed_dataset_csv", + "seed_dataset_json", + "seed_dataset_jsonl", + ], +) +def test_seed_dataset_generator_index_range_selection_strategy(fixture_name, stub_resource_provider, request): + """Test that generator correctly applies index range selection strategy.""" + # Ordered Sampling + + # Range with a subset of items + file_path = request.getfixturevalue(fixture_name) + generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.ORDERED, selection_strategy=IndexRange(start=4, end=8)) + result = generator.generate_from_scratch(6) + assert len(result) == 6 + assert list(result["name"]) == ["Eve", "Frank", "Grace", "Henry", "Ivy", "Eve"] + + # Range with just one item + generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.ORDERED, selection_strategy=IndexRange(start=4, end=4)) + result = generator.generate_from_scratch(1) + assert len(result) == 1 + assert list(result["name"]) == ["Eve"] + + # Range with all items + generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.ORDERED, selection_strategy=IndexRange(start=0, end=9)) + result = generator.generate_from_scratch(10) + assert len(result) == 10 + assert list(result["name"]) == ["Alice", "Bob", "Charlie", "David", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack"] + + # Shuffle Sampling + + # Range with a subset of items + generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.SHUFFLE, selection_strategy=IndexRange(start=4, end=8)) + result = generator.generate_from_scratch(10) + assert len(result) == 10 + assert set(result["name"]).issubset({"Eve", "Frank", "Grace", "Henry", "Ivy"}) + + # Range with just one item + generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.SHUFFLE, selection_strategy=IndexRange(start=4, end=4)) + result = generator.generate_from_scratch(1) + assert len(result) == 1 + assert list(result["name"]) == ["Eve"] + + # Range with all items + generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.SHUFFLE, selection_strategy=IndexRange(start=0, end=9)) + result = generator.generate_from_scratch(10) + assert len(result) == 10 + assert set(result["name"]).issubset({"Alice", "Bob", "Charlie", "David", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack"}) + + +@pytest.mark.parametrize( + "fixture_name", + [ + "seed_dataset_parquet", + "seed_dataset_csv", + "seed_dataset_json", + "seed_dataset_jsonl", + ], +) +def test_seed_dataset_generator_partition_block_selection_strategy(fixture_name, stub_resource_provider, request): + """Test that generator correctly applies partition block selection strategy.""" + file_path = request.getfixturevalue(fixture_name) + generator = create_generator_with_real_file( + file_path, + stub_resource_provider, + sampling_strategy=SamplingStrategy.ORDERED, + selection_strategy=PartitionBlock(partition_index=1, num_partitions=3) + ) + result = generator.generate_from_scratch(5) + assert len(result) == 5 + # Requesting 5 items from a 3-item partition should cycle: + assert list(result["name"]) == ["David", "Eve", "Frank", "David", "Eve"] + + generator = create_generator_with_real_file( + file_path, + stub_resource_provider, + sampling_strategy=SamplingStrategy.SHUFFLE, + selection_strategy=PartitionBlock(partition_index=4, num_partitions=5)) + result = generator.generate_from_scratch(10) + assert len(result) == 10 + assert set(result["name"]).issubset({"Jack", "Ivy"}) + + +@pytest.mark.parametrize( + "fixture_name", + [ + "seed_dataset_parquet", + "seed_dataset_csv", + "seed_dataset_json", + "seed_dataset_jsonl", + ], +) +def test_seed_dataset_generator_invalid_selection_strategies(fixture_name, stub_resource_provider, request): + """Test that generator raises an error for invalid selection strategies.""" + file_path = request.getfixturevalue(fixture_name) + with pytest.raises(SeedDatasetError, match="Selection strategy 'end' index 10 is out of bounds for dataset size 10"): + generator = create_generator_with_real_file(file_path, stub_resource_provider, selection_strategy=IndexRange(start=1, end=10)) + generator.generate_from_scratch(1) + with pytest.raises(SeedDatasetError, match="Selection strategy 'num_partitions' 11 is out of bounds for dataset size 10"): + generator = create_generator_with_real_file(file_path, stub_resource_provider, selection_strategy=PartitionBlock(partition_index=0, num_partitions=11)) + generator.generate_from_scratch(1) From 6bd77607204de2ddd959a033f963e7dd356c227d Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Mon, 3 Nov 2025 16:10:28 -0700 Subject: [PATCH 04/21] linting --- .../generators/seed_dataset.py | 16 ++-- tests/config/test_seed.py | 7 +- .../generators/test_seed_dataset.py | 76 ++++++++++++++----- 3 files changed, 73 insertions(+), 26 deletions(-) diff --git a/src/data_designer/engine/column_generators/generators/seed_dataset.py b/src/data_designer/engine/column_generators/generators/seed_dataset.py index 87066abcc..8b700e7c4 100644 --- a/src/data_designer/engine/column_generators/generators/seed_dataset.py +++ b/src/data_designer/engine/column_generators/generators/seed_dataset.py @@ -7,7 +7,7 @@ import duckdb import pandas as pd -from data_designer.config.seed import SamplingStrategy, IndexRange, PartitionBlock +from data_designer.config.seed import IndexRange, PartitionBlock, SamplingStrategy from data_designer.engine.column_generators.generators.base import ( FromScratchColumnGenerator, GenerationStrategy, @@ -64,9 +64,15 @@ def _initialize(self) -> None: def _validate_selection_strategy(self) -> None: err_msg = None if self.config.selection_strategy is not None: - if isinstance(self.config.selection_strategy, IndexRange) and self.config.selection_strategy.end >= self._seed_dataset_size: + if ( + isinstance(self.config.selection_strategy, IndexRange) + and self.config.selection_strategy.end >= self._seed_dataset_size + ): err_msg = f"Selection strategy 'end' index {self.config.selection_strategy.end} is out of bounds for dataset size {self._seed_dataset_size}" - elif isinstance(self.config.selection_strategy, PartitionBlock) and self.config.selection_strategy.num_partitions > self._seed_dataset_size: + elif ( + isinstance(self.config.selection_strategy, PartitionBlock) + and self.config.selection_strategy.num_partitions > self._seed_dataset_size + ): err_msg = f"Selection strategy 'num_partitions' {self.config.selection_strategy.num_partitions} is out of bounds for dataset size {self._seed_dataset_size}" if err_msg is not None: raise SeedDatasetError(err_msg) @@ -102,9 +108,7 @@ def _reset_batch_reader(self, num_records: int) -> None: else: read_query = f"SELECT * FROM '{self._dataset_uri}'{shuffle_query}" - self._batch_reader = self.duckdb_conn.query(read_query).record_batch( - batch_size=num_records - ) + self._batch_reader = self.duckdb_conn.query(read_query).record_batch(batch_size=num_records) def _sample_records(self, num_records: int) -> pd.DataFrame: logger.info(f"🌱 Sampling {num_records} records from seed dataset") diff --git a/tests/config/test_seed.py b/tests/config/test_seed.py index dddeda4ef..01775c466 100644 --- a/tests/config/test_seed.py +++ b/tests/config/test_seed.py @@ -2,10 +2,11 @@ from data_designer.config.seed import IndexRange, PartitionBlock + def test_index_range_validation(): with pytest.raises(ValueError, match="should be greater than or equal to 0"): IndexRange(start=-1, end=10) - + with pytest.raises(ValueError, match="should be greater than or equal to 0"): IndexRange(start=0, end=-1) @@ -22,10 +23,10 @@ def test_index_range_size(): def test_partition_block_validation(): with pytest.raises(ValueError, match="should be greater than or equal to 0"): PartitionBlock(partition_index=-1, num_partitions=10) - + with pytest.raises(ValueError, match="should be greater than or equal to 1"): PartitionBlock(partition_index=0, num_partitions=0) - + with pytest.raises(ValueError, match="'partition_index' must be less than 'num_partitions'"): PartitionBlock(partition_index=10, num_partitions=10) diff --git a/tests/engine/column_generators/generators/test_seed_dataset.py b/tests/engine/column_generators/generators/test_seed_dataset.py index 3b43b7ef2..7265c5f5d 100644 --- a/tests/engine/column_generators/generators/test_seed_dataset.py +++ b/tests/engine/column_generators/generators/test_seed_dataset.py @@ -10,15 +10,15 @@ import pytest from data_designer.config.columns import SeedDatasetColumnConfig -from data_designer.config.seed import SamplingStrategy, IndexRange, PartitionBlock +from data_designer.config.seed import IndexRange, PartitionBlock, SamplingStrategy from data_designer.engine.column_generators.generators.base import GenerationStrategy from data_designer.engine.column_generators.generators.seed_dataset import ( MAX_ZERO_RECORD_RESPONSE_FACTOR, SeedDatasetColumnGenerator, ) -from data_designer.engine.dataset_builders.multi_column_configs import SeedDatasetMultiColumnConfig -from data_designer.engine.resources.resource_provider import ResourceType, ResourceProvider from data_designer.engine.column_generators.utils.errors import SeedDatasetError +from data_designer.engine.dataset_builders.multi_column_configs import SeedDatasetMultiColumnConfig +from data_designer.engine.resources.resource_provider import ResourceProvider, ResourceType @pytest.fixture @@ -338,7 +338,8 @@ def create_generator_with_real_file( file_path: str, stub_resource_provider: ResourceProvider, sampling_strategy: SamplingStrategy = SamplingStrategy.ORDERED, - selection_strategy: IndexRange | PartitionBlock | None = None) -> SeedDatasetColumnGenerator: + selection_strategy: IndexRange | PartitionBlock | None = None, +) -> SeedDatasetColumnGenerator: """Helper function to create a generator with a real file and DuckDB connection.""" config = SeedDatasetMultiColumnConfig( columns=[ @@ -631,19 +632,34 @@ def test_seed_dataset_generator_index_range_selection_strategy(fixture_name, stu # Range with a subset of items file_path = request.getfixturevalue(fixture_name) - generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.ORDERED, selection_strategy=IndexRange(start=4, end=8)) + generator = create_generator_with_real_file( + file_path, + stub_resource_provider, + sampling_strategy=SamplingStrategy.ORDERED, + selection_strategy=IndexRange(start=4, end=8), + ) result = generator.generate_from_scratch(6) assert len(result) == 6 assert list(result["name"]) == ["Eve", "Frank", "Grace", "Henry", "Ivy", "Eve"] # Range with just one item - generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.ORDERED, selection_strategy=IndexRange(start=4, end=4)) + generator = create_generator_with_real_file( + file_path, + stub_resource_provider, + sampling_strategy=SamplingStrategy.ORDERED, + selection_strategy=IndexRange(start=4, end=4), + ) result = generator.generate_from_scratch(1) assert len(result) == 1 assert list(result["name"]) == ["Eve"] # Range with all items - generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.ORDERED, selection_strategy=IndexRange(start=0, end=9)) + generator = create_generator_with_real_file( + file_path, + stub_resource_provider, + sampling_strategy=SamplingStrategy.ORDERED, + selection_strategy=IndexRange(start=0, end=9), + ) result = generator.generate_from_scratch(10) assert len(result) == 10 assert list(result["name"]) == ["Alice", "Bob", "Charlie", "David", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack"] @@ -651,22 +667,39 @@ def test_seed_dataset_generator_index_range_selection_strategy(fixture_name, stu # Shuffle Sampling # Range with a subset of items - generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.SHUFFLE, selection_strategy=IndexRange(start=4, end=8)) + generator = create_generator_with_real_file( + file_path, + stub_resource_provider, + sampling_strategy=SamplingStrategy.SHUFFLE, + selection_strategy=IndexRange(start=4, end=8), + ) result = generator.generate_from_scratch(10) assert len(result) == 10 assert set(result["name"]).issubset({"Eve", "Frank", "Grace", "Henry", "Ivy"}) # Range with just one item - generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.SHUFFLE, selection_strategy=IndexRange(start=4, end=4)) + generator = create_generator_with_real_file( + file_path, + stub_resource_provider, + sampling_strategy=SamplingStrategy.SHUFFLE, + selection_strategy=IndexRange(start=4, end=4), + ) result = generator.generate_from_scratch(1) assert len(result) == 1 assert list(result["name"]) == ["Eve"] # Range with all items - generator = create_generator_with_real_file(file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.SHUFFLE, selection_strategy=IndexRange(start=0, end=9)) + generator = create_generator_with_real_file( + file_path, + stub_resource_provider, + sampling_strategy=SamplingStrategy.SHUFFLE, + selection_strategy=IndexRange(start=0, end=9), + ) result = generator.generate_from_scratch(10) assert len(result) == 10 - assert set(result["name"]).issubset({"Alice", "Bob", "Charlie", "David", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack"}) + assert set(result["name"]).issubset( + {"Alice", "Bob", "Charlie", "David", "Eve", "Frank", "Grace", "Henry", "Ivy", "Jack"} + ) @pytest.mark.parametrize( @@ -685,7 +718,7 @@ def test_seed_dataset_generator_partition_block_selection_strategy(fixture_name, file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.ORDERED, - selection_strategy=PartitionBlock(partition_index=1, num_partitions=3) + selection_strategy=PartitionBlock(partition_index=1, num_partitions=3), ) result = generator.generate_from_scratch(5) assert len(result) == 5 @@ -696,7 +729,8 @@ def test_seed_dataset_generator_partition_block_selection_strategy(fixture_name, file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.SHUFFLE, - selection_strategy=PartitionBlock(partition_index=4, num_partitions=5)) + selection_strategy=PartitionBlock(partition_index=4, num_partitions=5), + ) result = generator.generate_from_scratch(10) assert len(result) == 10 assert set(result["name"]).issubset({"Jack", "Ivy"}) @@ -714,9 +748,17 @@ def test_seed_dataset_generator_partition_block_selection_strategy(fixture_name, def test_seed_dataset_generator_invalid_selection_strategies(fixture_name, stub_resource_provider, request): """Test that generator raises an error for invalid selection strategies.""" file_path = request.getfixturevalue(fixture_name) - with pytest.raises(SeedDatasetError, match="Selection strategy 'end' index 10 is out of bounds for dataset size 10"): - generator = create_generator_with_real_file(file_path, stub_resource_provider, selection_strategy=IndexRange(start=1, end=10)) + with pytest.raises( + SeedDatasetError, match="Selection strategy 'end' index 10 is out of bounds for dataset size 10" + ): + generator = create_generator_with_real_file( + file_path, stub_resource_provider, selection_strategy=IndexRange(start=1, end=10) + ) generator.generate_from_scratch(1) - with pytest.raises(SeedDatasetError, match="Selection strategy 'num_partitions' 11 is out of bounds for dataset size 10"): - generator = create_generator_with_real_file(file_path, stub_resource_provider, selection_strategy=PartitionBlock(partition_index=0, num_partitions=11)) + with pytest.raises( + SeedDatasetError, match="Selection strategy 'num_partitions' 11 is out of bounds for dataset size 10" + ): + generator = create_generator_with_real_file( + file_path, stub_resource_provider, selection_strategy=PartitionBlock(partition_index=0, num_partitions=11) + ) generator.generate_from_scratch(1) From 12301b3d789088935d93c2bc65dc83a93b5b455f Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Mon, 3 Nov 2025 17:14:19 -0700 Subject: [PATCH 05/21] update tests + test e2e --- src/data_designer/config/config_builder.py | 9 ++++--- .../generators/seed_dataset.py | 5 ++-- .../dataset_builders/utils/config_compiler.py | 1 + .../generators/test_seed_dataset.py | 24 +++++++++++++++++++ 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/data_designer/config/config_builder.py b/src/data_designer/config/config_builder.py index 78b2195da..0e40bd03d 100644 --- a/src/data_designer/config/config_builder.py +++ b/src/data_designer/config/config_builder.py @@ -6,7 +6,7 @@ import json import logging from pathlib import Path - +from typing import Union, Optional from pygments import highlight from pygments.formatters import HtmlFormatter from pygments.lexers import PythonLexer @@ -31,6 +31,8 @@ SamplingStrategy, SeedConfig, SeedDatasetReference, + IndexRange, + PartitionBlock, ) from .utils.constants import DEFAULT_REPR_HTML_STYLE, REPR_HTML_TEMPLATE from .utils.info import DataDesignerInfo @@ -113,7 +115,7 @@ def from_config(cls, config: dict | str | Path | BuilderConfig) -> Self: datastore_settings=builder_config.datastore_settings, ) builder.set_seed_datastore_settings(builder_config.datastore_settings) - builder.with_seed_dataset(seed_dataset_reference, sampling_strategy=config.seed_config.sampling_strategy) + builder.with_seed_dataset(seed_dataset_reference, sampling_strategy=config.seed_config.sampling_strategy, selection_strategy=config.seed_config.selection_strategy) return builder @@ -493,6 +495,7 @@ def with_seed_dataset( dataset_reference: SeedDatasetReference, *, sampling_strategy: SamplingStrategy = SamplingStrategy.ORDERED, + selection_strategy: Optional[Union[IndexRange, PartitionBlock]] = None, ) -> Self: """Add a seed dataset to the current Data Designer configuration. @@ -508,7 +511,7 @@ def with_seed_dataset( Returns: The current Data Designer config builder instance. """ - self._seed_config = SeedConfig(dataset=dataset_reference.dataset, sampling_strategy=sampling_strategy) + self._seed_config = SeedConfig(dataset=dataset_reference.dataset, sampling_strategy=sampling_strategy, selection_strategy=selection_strategy) self.set_seed_datastore_settings( dataset_reference.datastore_settings if hasattr(dataset_reference, "datastore_settings") else None ) diff --git a/src/data_designer/engine/column_generators/generators/seed_dataset.py b/src/data_designer/engine/column_generators/generators/seed_dataset.py index 8b700e7c4..2579b82b0 100644 --- a/src/data_designer/engine/column_generators/generators/seed_dataset.py +++ b/src/data_designer/engine/column_generators/generators/seed_dataset.py @@ -60,7 +60,7 @@ def _initialize(self) -> None: self._dataset_uri = self.resource_provider.datastore.get_dataset_uri(self.config.dataset) self._seed_dataset_size = self.duckdb_conn.execute(f"SELECT COUNT(*) FROM '{self._dataset_uri}'").fetchone()[0] self._index_range = self._resolve_index_range() - + def _validate_selection_strategy(self) -> None: err_msg = None if self.config.selection_strategy is not None: @@ -115,9 +115,8 @@ def _sample_records(self, num_records: int) -> pd.DataFrame: logger.info(f" |-- seed dataset size: {self._seed_dataset_size} records") logger.info(f" |-- sampling strategy: {self.config.sampling_strategy}") if self._index_range is not None: - logger.info(f" |-- selection strategy: {self.config.selection_strategy.model_dump_json()}") + logger.info(f" |-- selection strategy: {type(self.config.selection_strategy).__name__}\n{self.config.selection_strategy.model_dump_json(indent=4)}") logger.info(f" |-- seed dataset size after selection: {self._index_range.size} records") - df_batch = pd.DataFrame() df_sample = pd.DataFrame() if self._df_remaining is None else self._df_remaining num_zero_record_responses = 0 diff --git a/src/data_designer/engine/dataset_builders/utils/config_compiler.py b/src/data_designer/engine/dataset_builders/utils/config_compiler.py index e302a99e6..7a94c4ecf 100644 --- a/src/data_designer/engine/dataset_builders/utils/config_compiler.py +++ b/src/data_designer/engine/dataset_builders/utils/config_compiler.py @@ -35,6 +35,7 @@ def compile_dataset_builder_column_configs(config: DataDesignerConfig) -> list[D columns=seed_column_configs, dataset=config.seed_config.dataset, sampling_strategy=config.seed_config.sampling_strategy, + selection_strategy=config.seed_config.selection_strategy, ) ) diff --git a/tests/engine/column_generators/generators/test_seed_dataset.py b/tests/engine/column_generators/generators/test_seed_dataset.py index 7265c5f5d..bf2d92ae2 100644 --- a/tests/engine/column_generators/generators/test_seed_dataset.py +++ b/tests/engine/column_generators/generators/test_seed_dataset.py @@ -124,6 +124,30 @@ def test_seed_dataset_column_generator_config_structure(): assert config.columns[0].column_type.value == "seed-dataset" assert config.columns[1].name == "col2" assert config.columns[1].column_type.value == "seed-dataset" + assert config.selection_strategy is None + + # Test PartitionBlock selection strategy + config = SeedDatasetMultiColumnConfig( + columns=[SeedDatasetColumnConfig(name="col1"), SeedDatasetColumnConfig(name="col2")], + dataset="test/dataset", + sampling_strategy=SamplingStrategy.SHUFFLE, + selection_strategy=PartitionBlock(partition_index=1, num_partitions=3), + ) + assert isinstance(config.selection_strategy, PartitionBlock) + assert config.selection_strategy.partition_index == 1 + assert config.selection_strategy.num_partitions == 3 + + # Test IndexRange selection strategy + config = SeedDatasetMultiColumnConfig( + columns=[SeedDatasetColumnConfig(name="col1"), SeedDatasetColumnConfig(name="col2")], + dataset="test/dataset", + sampling_strategy=SamplingStrategy.SHUFFLE, + selection_strategy=IndexRange(start=0, end=1), + ) + assert isinstance(config.selection_strategy, IndexRange) + assert config.selection_strategy.start == 0 + assert config.selection_strategy.end == 1 + assert config.selection_strategy.size == 2 # Test constants and enum values assert MAX_ZERO_RECORD_RESPONSE_FACTOR == 2 From 03c048ed29782db2cec67bc24b0e63caf003e7e1 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Mon, 3 Nov 2025 17:25:46 -0700 Subject: [PATCH 06/21] run ruff --- src/data_designer/config/config_builder.py | 19 ++++++++++++++----- .../generators/seed_dataset.py | 6 ++++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/data_designer/config/config_builder.py b/src/data_designer/config/config_builder.py index 0e40bd03d..a6667f441 100644 --- a/src/data_designer/config/config_builder.py +++ b/src/data_designer/config/config_builder.py @@ -6,7 +6,8 @@ import json import logging from pathlib import Path -from typing import Union, Optional +from typing import Optional, Union + from pygments import highlight from pygments.formatters import HtmlFormatter from pygments.lexers import PythonLexer @@ -27,12 +28,12 @@ ) from .seed import ( DatastoreSeedDatasetReference, + IndexRange, LocalSeedDatasetReference, + PartitionBlock, SamplingStrategy, SeedConfig, SeedDatasetReference, - IndexRange, - PartitionBlock, ) from .utils.constants import DEFAULT_REPR_HTML_STYLE, REPR_HTML_TEMPLATE from .utils.info import DataDesignerInfo @@ -115,7 +116,11 @@ def from_config(cls, config: dict | str | Path | BuilderConfig) -> Self: datastore_settings=builder_config.datastore_settings, ) builder.set_seed_datastore_settings(builder_config.datastore_settings) - builder.with_seed_dataset(seed_dataset_reference, sampling_strategy=config.seed_config.sampling_strategy, selection_strategy=config.seed_config.selection_strategy) + builder.with_seed_dataset( + seed_dataset_reference, + sampling_strategy=config.seed_config.sampling_strategy, + selection_strategy=config.seed_config.selection_strategy, + ) return builder @@ -511,7 +516,11 @@ def with_seed_dataset( Returns: The current Data Designer config builder instance. """ - self._seed_config = SeedConfig(dataset=dataset_reference.dataset, sampling_strategy=sampling_strategy, selection_strategy=selection_strategy) + self._seed_config = SeedConfig( + dataset=dataset_reference.dataset, + sampling_strategy=sampling_strategy, + selection_strategy=selection_strategy, + ) self.set_seed_datastore_settings( dataset_reference.datastore_settings if hasattr(dataset_reference, "datastore_settings") else None ) diff --git a/src/data_designer/engine/column_generators/generators/seed_dataset.py b/src/data_designer/engine/column_generators/generators/seed_dataset.py index 2579b82b0..b33ace2f3 100644 --- a/src/data_designer/engine/column_generators/generators/seed_dataset.py +++ b/src/data_designer/engine/column_generators/generators/seed_dataset.py @@ -60,7 +60,7 @@ def _initialize(self) -> None: self._dataset_uri = self.resource_provider.datastore.get_dataset_uri(self.config.dataset) self._seed_dataset_size = self.duckdb_conn.execute(f"SELECT COUNT(*) FROM '{self._dataset_uri}'").fetchone()[0] self._index_range = self._resolve_index_range() - + def _validate_selection_strategy(self) -> None: err_msg = None if self.config.selection_strategy is not None: @@ -115,7 +115,9 @@ def _sample_records(self, num_records: int) -> pd.DataFrame: logger.info(f" |-- seed dataset size: {self._seed_dataset_size} records") logger.info(f" |-- sampling strategy: {self.config.sampling_strategy}") if self._index_range is not None: - logger.info(f" |-- selection strategy: {type(self.config.selection_strategy).__name__}\n{self.config.selection_strategy.model_dump_json(indent=4)}") + logger.info( + f" |-- selection strategy: {type(self.config.selection_strategy).__name__}\n{self.config.selection_strategy.model_dump_json(indent=4)}" + ) logger.info(f" |-- seed dataset size after selection: {self._index_range.size} records") df_batch = pd.DataFrame() df_sample = pd.DataFrame() if self._df_remaining is None else self._df_remaining From 6da2c3c16940cdcdd892ee406051f116e14ebfc0 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Mon, 3 Nov 2025 17:31:25 -0700 Subject: [PATCH 07/21] license check header --- tests/config/test_seed.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/config/test_seed.py b/tests/config/test_seed.py index 01775c466..8f999ccd4 100644 --- a/tests/config/test_seed.py +++ b/tests/config/test_seed.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import pytest from data_designer.config.seed import IndexRange, PartitionBlock From 80fca5172d294ab3ade398a2c07503ee0c53655b Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Mon, 3 Nov 2025 19:31:04 -0700 Subject: [PATCH 08/21] partition_index -> index --- src/data_designer/config/seed.py | 12 ++++++------ tests/config/test_seed.py | 16 ++++++++-------- .../generators/test_seed_dataset.py | 10 +++++----- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/data_designer/config/seed.py b/src/data_designer/config/seed.py index 9554a8503..5ee1453a3 100644 --- a/src/data_designer/config/seed.py +++ b/src/data_designer/config/seed.py @@ -34,24 +34,24 @@ def size(self) -> int: class PartitionBlock(ConfigBase): - partition_index: int = Field(default=0, ge=0, description="The index of the partition to sample from") + index: int = Field(default=0, ge=0, description="The index of the partition to sample from") num_partitions: int = Field(default=1, ge=1, description="The total number of partitions in the dataset") @model_validator(mode="after") def _validate_partition_block(self) -> Self: - if self.partition_index >= self.num_partitions: - raise ValueError("'partition_index' must be less than 'num_partitions'") + if self.index >= self.num_partitions: + raise ValueError("'index' must be less than 'num_partitions'") return self def to_index_range(self, dataset_size: int) -> IndexRange: partition_size = dataset_size // self.num_partitions - start = self.partition_index * partition_size + start = self.index * partition_size # For the last partition, extend to the end of the dataset to include remainder rows - if self.partition_index == self.num_partitions - 1: + if self.index == self.num_partitions - 1: end = dataset_size - 1 else: - end = ((self.partition_index + 1) * partition_size) - 1 + end = ((self.index + 1) * partition_size) - 1 return IndexRange(start=start, end=end) diff --git a/tests/config/test_seed.py b/tests/config/test_seed.py index 8f999ccd4..2e7a2e1b9 100644 --- a/tests/config/test_seed.py +++ b/tests/config/test_seed.py @@ -25,32 +25,32 @@ def test_index_range_size(): def test_partition_block_validation(): with pytest.raises(ValueError, match="should be greater than or equal to 0"): - PartitionBlock(partition_index=-1, num_partitions=10) + PartitionBlock(index=-1, num_partitions=10) with pytest.raises(ValueError, match="should be greater than or equal to 1"): - PartitionBlock(partition_index=0, num_partitions=0) + PartitionBlock(index=0, num_partitions=0) - with pytest.raises(ValueError, match="'partition_index' must be less than 'num_partitions'"): - PartitionBlock(partition_index=10, num_partitions=10) + with pytest.raises(ValueError, match="'index' must be less than 'num_partitions'"): + PartitionBlock(index=10, num_partitions=10) def test_partition_block_to_index_range(): - index_range = PartitionBlock(partition_index=0, num_partitions=10).to_index_range(101) + index_range = PartitionBlock(index=0, num_partitions=10).to_index_range(101) assert index_range.start == 0 assert index_range.end == 9 assert index_range.size == 10 - index_range = PartitionBlock(partition_index=1, num_partitions=10).to_index_range(105) + index_range = PartitionBlock(index=1, num_partitions=10).to_index_range(105) assert index_range.start == 10 assert index_range.end == 19 assert index_range.size == 10 - index_range = PartitionBlock(partition_index=2, num_partitions=10).to_index_range(105) + index_range = PartitionBlock(index=2, num_partitions=10).to_index_range(105) assert index_range.start == 20 assert index_range.end == 29 assert index_range.size == 10 - index_range = PartitionBlock(partition_index=9, num_partitions=10).to_index_range(105) + index_range = PartitionBlock(index=9, num_partitions=10).to_index_range(105) assert index_range.start == 90 assert index_range.end == 104 assert index_range.size == 15 diff --git a/tests/engine/column_generators/generators/test_seed_dataset.py b/tests/engine/column_generators/generators/test_seed_dataset.py index bf2d92ae2..487d143f4 100644 --- a/tests/engine/column_generators/generators/test_seed_dataset.py +++ b/tests/engine/column_generators/generators/test_seed_dataset.py @@ -131,10 +131,10 @@ def test_seed_dataset_column_generator_config_structure(): columns=[SeedDatasetColumnConfig(name="col1"), SeedDatasetColumnConfig(name="col2")], dataset="test/dataset", sampling_strategy=SamplingStrategy.SHUFFLE, - selection_strategy=PartitionBlock(partition_index=1, num_partitions=3), + selection_strategy=PartitionBlock(index=1, num_partitions=3), ) assert isinstance(config.selection_strategy, PartitionBlock) - assert config.selection_strategy.partition_index == 1 + assert config.selection_strategy.index == 1 assert config.selection_strategy.num_partitions == 3 # Test IndexRange selection strategy @@ -742,7 +742,7 @@ def test_seed_dataset_generator_partition_block_selection_strategy(fixture_name, file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.ORDERED, - selection_strategy=PartitionBlock(partition_index=1, num_partitions=3), + selection_strategy=PartitionBlock(index=1, num_partitions=3), ) result = generator.generate_from_scratch(5) assert len(result) == 5 @@ -753,7 +753,7 @@ def test_seed_dataset_generator_partition_block_selection_strategy(fixture_name, file_path, stub_resource_provider, sampling_strategy=SamplingStrategy.SHUFFLE, - selection_strategy=PartitionBlock(partition_index=4, num_partitions=5), + selection_strategy=PartitionBlock(index=4, num_partitions=5), ) result = generator.generate_from_scratch(10) assert len(result) == 10 @@ -783,6 +783,6 @@ def test_seed_dataset_generator_invalid_selection_strategies(fixture_name, stub_ SeedDatasetError, match="Selection strategy 'num_partitions' 11 is out of bounds for dataset size 10" ): generator = create_generator_with_real_file( - file_path, stub_resource_provider, selection_strategy=PartitionBlock(partition_index=0, num_partitions=11) + file_path, stub_resource_provider, selection_strategy=PartitionBlock(index=0, num_partitions=11) ) generator.generate_from_scratch(1) From 4e3bd3edca407e3d9e5cd9a578f25e5be95e3232 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Mon, 3 Nov 2025 19:38:12 -0700 Subject: [PATCH 09/21] update log message --- .../engine/column_generators/generators/seed_dataset.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/data_designer/engine/column_generators/generators/seed_dataset.py b/src/data_designer/engine/column_generators/generators/seed_dataset.py index b33ace2f3..277782543 100644 --- a/src/data_designer/engine/column_generators/generators/seed_dataset.py +++ b/src/data_designer/engine/column_generators/generators/seed_dataset.py @@ -115,9 +115,12 @@ def _sample_records(self, num_records: int) -> pd.DataFrame: logger.info(f" |-- seed dataset size: {self._seed_dataset_size} records") logger.info(f" |-- sampling strategy: {self.config.sampling_strategy}") if self._index_range is not None: - logger.info( - f" |-- selection strategy: {type(self.config.selection_strategy).__name__}\n{self.config.selection_strategy.model_dump_json(indent=4)}" - ) + if isinstance(self.config.selection_strategy, IndexRange): + logger.info(f" |-- selection: rows [{self._index_range.start} to {self._index_range.end}] inclusive") + else: + logger.info( + f" |-- selection: partition {self.config.selection_strategy.index + 1} of {self.config.selection_strategy.num_partitions}" + ) logger.info(f" |-- seed dataset size after selection: {self._index_range.size} records") df_batch = pd.DataFrame() df_sample = pd.DataFrame() if self._df_remaining is None else self._df_remaining From c32b278887682d6d1f286982e91ea5359ae5e531 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Mon, 3 Nov 2025 19:41:04 -0700 Subject: [PATCH 10/21] Remove sub subquery alias notneeded --- .../engine/column_generators/generators/seed_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data_designer/engine/column_generators/generators/seed_dataset.py b/src/data_designer/engine/column_generators/generators/seed_dataset.py index 277782543..9d9c5ecb2 100644 --- a/src/data_designer/engine/column_generators/generators/seed_dataset.py +++ b/src/data_designer/engine/column_generators/generators/seed_dataset.py @@ -101,7 +101,7 @@ def _reset_batch_reader(self, num_records: int) -> None: SELECT * EXCLUDE (row_num) FROM ( SELECT *, row_number() OVER () as row_num FROM '{self._dataset_uri}' - ) sub + ) WHERE row_num > {self._index_range.start} AND row_num <= {self._index_range.end + 1} {shuffle_query} """ From 483363d480282f17033b77b7f03dae751dae0c76 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Tue, 4 Nov 2025 12:56:43 -0700 Subject: [PATCH 11/21] Optimize duckdb seed dataset select based on on limit and offset --- .../generators/seed_dataset.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/data_designer/engine/column_generators/generators/seed_dataset.py b/src/data_designer/engine/column_generators/generators/seed_dataset.py index 9d9c5ecb2..355786029 100644 --- a/src/data_designer/engine/column_generators/generators/seed_dataset.py +++ b/src/data_designer/engine/column_generators/generators/seed_dataset.py @@ -92,22 +92,20 @@ def _reset_batch_reader(self, num_records: int) -> None: shuffle_query = " ORDER BY RANDOM()" if shuffle else "" if self._index_range is not None: - # Use subquery with row_number() window function to filter by index range - # IndexRange uses 0-based indexing [start, end] inclusive, row_number() is 1-based - # To convert 0-based index i to 1-based row_number: row_number = i + 1 - # For inclusive range [start, end], we want: row_number > start AND row_number <= end + 1 - # This gives us 1-based rows [start+1, end+1] which maps to 0-based indices [start, end] + # Use LIMIT and OFFSET for efficient index range filtering + # IndexRange uses 0-based indexing [start, end] inclusive + # OFFSET skips the first 'start' rows (0-based) + # LIMIT takes 'end - start + 1' rows to include both start and end (inclusive) + offset_value = self._index_range.start + limit_value = self._index_range.end - self._index_range.start + 1 read_query = f""" - SELECT * EXCLUDE (row_num) FROM ( - SELECT *, row_number() OVER () as row_num - FROM '{self._dataset_uri}' - ) - WHERE row_num > {self._index_range.start} AND row_num <= {self._index_range.end + 1} - {shuffle_query} + SELECT * FROM '{self._dataset_uri}' + LIMIT {limit_value} OFFSET {offset_value} """ + + read_query = f"SELECT * FROM ({read_query}){shuffle_query}" else: read_query = f"SELECT * FROM '{self._dataset_uri}'{shuffle_query}" - self._batch_reader = self.duckdb_conn.query(read_query).record_batch(batch_size=num_records) def _sample_records(self, num_records: int) -> pd.DataFrame: From 976564365cacc96ad4fed2560e0ee23d4d030abc Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Tue, 4 Nov 2025 13:30:18 -0700 Subject: [PATCH 12/21] Add docstring to seedconfig --- src/data_designer/config/seed.py | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/data_designer/config/seed.py b/src/data_designer/config/seed.py index 5ee1453a3..a467b98a9 100644 --- a/src/data_designer/config/seed.py +++ b/src/data_designer/config/seed.py @@ -56,6 +56,56 @@ def to_index_range(self, dataset_size: int) -> IndexRange: class SeedConfig(ConfigBase): + """Configuration for sampling data from a seed dataset. + + Args: + dataset: Path or identifier for the seed dataset. + sampling_strategy: Strategy for how to sample rows from the dataset. + - ORDERED: Read rows sequentially in their original order. + - SHUFFLE: Randomly shuffle rows before sampling. When used with + selection_strategy, shuffling occurs within the selected range/partition. + selection_strategy: Optional strategy to select a subset of the dataset. + - IndexRange: Select a specific range of indices (e.g., rows 100-200). + - PartitionBlock: Select a partition by splitting the dataset into N equal parts. + Partition indices are zero-based (index=0 is the first partition, index=1 is + the second, etc.). + + Examples: + Read rows sequentially from start to end: + SeedConfig(dataset="my_data.parquet", sampling_strategy=SamplingStrategy.ORDERED) + + Read rows in random order: + SeedConfig(dataset="my_data.parquet", sampling_strategy=SamplingStrategy.SHUFFLE) + + Read specific index range (rows 100-199): + SeedConfig( + dataset="my_data.parquet", + sampling_strategy=SamplingStrategy.ORDERED, + selection_strategy=IndexRange(start=100, end=199) + ) + + Read random rows from a specific index range (shuffles within rows 100-199): + SeedConfig( + dataset="my_data.parquet", + sampling_strategy=SamplingStrategy.SHUFFLE, + selection_strategy=IndexRange(start=100, end=199) + ) + + Read from partition 2 (3rd partition, zero-based) of 5 partitions (20% of dataset): + SeedConfig( + dataset="my_data.parquet", + sampling_strategy=SamplingStrategy.ORDERED, + selection_strategy=PartitionBlock(index=2, num_partitions=5) + ) + + Read shuffled rows from partition 0 of 10 partitions (shuffles within the partition): + SeedConfig( + dataset="my_data.parquet", + sampling_strategy=SamplingStrategy.SHUFFLE, + selection_strategy=PartitionBlock(index=0, num_partitions=10) + ) + """ + dataset: str sampling_strategy: SamplingStrategy = SamplingStrategy.ORDERED selection_strategy: Optional[Union[IndexRange, PartitionBlock]] = None From 98993de715876abeab934dd0ddfb3e08ac43a3ca Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 30 Oct 2025 18:04:52 -0400 Subject: [PATCH 13/21] add guide --- CONTRIBUTING.md | 254 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..84aaa9f1a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,254 @@ +# 🎨✨ Contributing to NeMo Data Designer 🎨✨ + +Thank you for your interest in contributing to Data Designer! + +We welcome contributions from the community and sincerely appreciate your efforts to improve the project. Whether you're fixing a typo, reporting a bug, proposing a new feature, or implementing a major enhancement, your work helps make Data Designer better for everyone πŸŽ‰. + +This guide will help you get started with the contribution process. + +## Table of Contents + +- [Getting Started](#getting-started) +- [Ways to Contribute](#ways-to-contribute) +- [Feature Requests](#feature-requests) +- [Development Guide](#development-guide) +- [Code Quality Standards](#code-quality-standards) +- [Submitting Changes](#submitting-changes) +- [Code of Conduct](#code-of-conduct) +- [Signing off on your work](#signing-off-on-your-work) + + +## Getting Started +πŸ‘‹ Welcome to the Data Designer community! We're excited to have you here. + +Whether you're new to the project or ready to dive in, the resources below will help you get oriented and productive quickly: + +1. **[README.md](README.md)** – best place to start to learn the basics of the project + +2. **[AGENTS.md](AGENTS.md)** – code structure and development guidelines for humans and agents + +3. **[Documentation](docs/)** – detailed documentation on Data Designer's capabilities and usage + +## Ways to Contribute + +There are many ways to contribute to Data Designer: + +### πŸ› Bug Fixes + +Found a bug? Before reporting, please +1. Verify you're using the latest version: `uv pip install --upgrade data-designer` +2. Search for duplicates in the [issue tracker](https://github.com/NVIDIA-NeMo/DataDesigner/issues) + +When [creating a bug report](https://github.com/NVIDIA-NeMo/DataDesigner/issues/new), please include: +- Data Designer version: `python -c "import data_designer; print(data_designer.__version__)"` +- Python version and operating system +- Minimal reproducible example +- Expected vs. actual behavior +- Full error messages and stack traces + +If you are interested in fixing the bug yourself, that's AWESOME! Please follow the [development guide](#development-guide) to get started. + +### ✨ Feature Implementation +Want to add new functionality? Great! Please review our [development philosophy](#development-philosophy-push-extend-integrate) and [open a feature request](#feature-requests) first to discuss the approach before investing significant time in implementation. + +### πŸ“– Documentation Improvements +Documentation is crucial for user adoption. Contributions that clarify usage, add examples, or fix typos are highly valued. + +### πŸ’‘ Examples and Tutorials +Share your use cases! Example notebooks and tutorials help others understand how to leverage Data Designer effectively. + +### πŸ§ͺ Test Coverage +Help us improve test coverage by adding tests for untested code paths or edge cases. + +## Feature Requests + +Data Designer is designed to be as flexible and extensible as possible, and we welcome your ideas for pushing its capabilities even further! To help us keep the core library focused and maintainable while supporting innovation, we follow a simple development philosophy: + +### Development Philosophy: Push, Extend, Integrate + +1. πŸ§— **Push the Limits**: Can your use case be achieved with current features? We've designed Data Designer to be composable – sometimes creative combinations of existing tools can accomplish what you need. Check out our examples or open an issue if you'd like help exploring this! + +2. πŸ”Œ **Extend with a plugin**: If existing features aren't quite enough, can your idea be implemented and shared as a plugin that extends the core library? + +3. βš™οΈ **Integrate with the core library**: If your feature idea or existing plugin is broadly useful and aligns with Data Designer's goals, we'd love to integrate it! We're happy to discuss whether it is a good fit for the core library. + +### Submitting a Feature Request + +Open a [new issue](https://github.com/NVIDIA-NeMo/DataDesigner/issues/new) with: + +- **Clear title**: Concise description of the feature +- **Use case**: Explain what problem this solves and why it's important +- **Proposed solution**: Describe how you envision the feature working +- **Alternatives considered**: Other approaches you've thought about +- **Examples**: Code examples or mockups of how users would interact with the feature +- **Willingness to implement**: Are you interested in implementing this yourself? + +## Development Guide + +Data Designer uses [`uv`](https://github.com/astral-sh/uv) for dependency management. If you don't have uv installed, follow their [installation instructions](https://docs.astral.sh/uv/getting-started/installation/). + +### Initial Setup + +0. **Create or find an issue** + + Before starting work, ensure there's an issue tracking your contribution: + - For bug fixes: Search [existing issues](https://github.com/NVIDIA-NeMo/DataDesigner/issues) or [create a new one](https://github.com/NVIDIA-NeMo/DataDesigner/issues/new) + - For new features: Open a [feature request](#feature-requests) to discuss the approach first + - Comment on the issue to let maintainers know you're working on it + +1. **Fork and clone the repository** + + Start by [forking the Data Designer repository](https://github.com/NVIDIA-NeMo/DataDesigner/fork), then clone your fork and add the upstream remote: + + ```bash + git clone https://github.com/YOUR_GITHUB_USERNAME/DataDesigner.git + + cd DataDesigner + + git remote add upstream https://github.com/NVIDIA-NeMo/DataDesigner.git + ``` + +2. **Install dependencies** + + ```bash + # Install project with dev dependencies + make install-dev + + # Or, if you use Jupyter / IPython for development + make install-dev-notebooks + ``` + + This creates a virtual environment in `.venv`. Activate it with: + + ```bash + source .venv/bin/activate + ``` + +3. **Verify your setup** + + ```bash + make test && make check-all + ``` + + If no errors are reported, you're ready to develop πŸš€ + +### Making Changes + +1. **Create a feature branch** + + ```bash + git checkout main + git pull upstream main + git checkout -b //#- + ``` + + Example types of change: + - `feat` for new features + - `fix` for bug fixes + - `docs` for documentation updates + - `test` for testing changes + - `refactor` for code refactoring + - `chore` for chore tasks + - `style` for style changes + - `perf` for performance improvements + + Example branch name: + - `feat/johnnygreco/#123-add-xyz-generator` for a new feature by @johnnygreco, addressing issue #123 + +2. **Develop your changes** + + Please follow the patterns and conventions outlined in [AGENTS.md](AGENTS.md). + +3. **Test and validate** + + ```bash + make check-all-fix # Format code and fix linting issues + make test # Run all tests + make coverage # Check test coverage (must be >90%) + ``` + + **Writing tests**: Place tests in [tests/](tests/) mirroring the source structure. Use fixtures from [tests/conftest.py](tests/conftest.py), mock external services with `unittest.mock` or `pytest-httpx`, and test both success and failure cases. See [AGENTS.md](AGENTS.md) for patterns and examples. + +4. **Commit your work** + + Write clear, descriptive commit messages, optionally including a brief summary (50 characters or less) and reference issue numbers when applicable (e.g., "Fixes #123"). + + ```bash + git commit -m "Add XYZ generator for synthetic data" -m "Fixes #123" + ``` + +5. **Stay up to date** + + Regularly sync your branch with upstream changes: + + ```bash + git fetch upstream + git merge upstream/main + ``` + +## Submitting Changes + +### Before Submitting + +Ensure your changes meet the following criteria: + +- Code follows patterns and conventions outlined in [AGENTS.md](AGENTS.md) +- All tests pass (`make test`) +- Code is formatted and linted (`make check-all-fix`) +- New functionality includes tests +- Documentation is updated (README, docstrings, examples) +- License headers are present on all new files +- Commit messages are clear and descriptive + +### Creating a Pull Request + +1. **Push your changes** to your fork: + + ```bash + git push origin //#- + ``` + +2. **Open a pull request** on GitHub from your fork to the main repository + +3. **Respond to review feedback** promptly and update your PR as needed + +### Pull Request Review Process + +- Maintainers will review your PR and may request changes +- Address feedback by pushing additional commits to your branch +- Once approved, a maintainer will merge your PR +- Your contribution will be included in the next release! + +## Code of Conduct + +Data Designer follows the Contributor Covenant Code of Conduct. We are committed to providing a welcoming and inclusive environment for all contributors. + +**Please read our complete [Code of Conduct](CODE_OF_CONDUCT.md)** for full details on our standards and expectations. + +### Copyright + +All code files that are added to this repository must include the appropriate NVIDIA copyright header: + +```python +# SPDX-FileCopyrightText: Copyright (c) {YEAR} NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +``` + +Use `make update-license-headers` to add headers automatically. + +## Getting Help + +Need help with your contribution? + +- **Documentation**: Check the [documentation](docs/) and [AGENTS.md](AGENTS.md) for additional information +- **Issues**: Browse [existing issues](https://github.com/NVIDIA-NeMo/DataDesigner/issues) for similar questions +- **Contact**: Reach out to the core maintainers at [data-designer@nvidia.com](mailto:data-designer@nvidia.com) + + +## Signing off on your work + +When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license. All contributors are asked to sign the Data Designer [Developer Certificate of Origin (DCO)](DCO) when submitting their first pull request. The process is automated by a bot that will comment on the pull request. Our DCO is the same as the Linux Foundation requires its contributors to sign. + +--- + +Thank you for contributing to NeMo Data Designer! Your efforts help make synthetic data generation more accessible and powerful for everyone. 🎨✨ From b4be82164ae1faf251d560a74119474fd7503e00 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 31 Oct 2025 10:09:57 -0400 Subject: [PATCH 14/21] feat req updates --- CONTRIBUTING.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 84aaa9f1a..70ed22a12 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,7 +49,7 @@ When [creating a bug report](https://github.com/NVIDIA-NeMo/DataDesigner/issues/ If you are interested in fixing the bug yourself, that's AWESOME! Please follow the [development guide](#development-guide) to get started. ### ✨ Feature Implementation -Want to add new functionality? Great! Please review our [development philosophy](#development-philosophy-push-extend-integrate) and [open a feature request](#feature-requests) first to discuss the approach before investing significant time in implementation. +Want to add new functionality? Great! Please review [our development approach](#feature-requests) and open a feature request to discuss the idea and get feedback before investing significant time on the implementation. ### πŸ“– Documentation Improvements Documentation is crucial for user adoption. Contributions that clarify usage, add examples, or fix typos are highly valued. @@ -62,15 +62,17 @@ Help us improve test coverage by adding tests for untested code paths or edge ca ## Feature Requests -Data Designer is designed to be as flexible and extensible as possible, and we welcome your ideas for pushing its capabilities even further! To help us keep the core library focused and maintainable while supporting innovation, we follow a simple development philosophy: +Data Designer is designed to be as flexible and extensible as possible, and we welcome your ideas for pushing its capabilities even further! To keep the core library maintainable, while also supporting innovation, we take an incremental approach: we explore what's already possible, extend through plugins when needed, and integrate the most broadly useful features into the core library: -### Development Philosophy: Push, Extend, Integrate +### How We Grow Data Designer -1. πŸ§— **Push the Limits**: Can your use case be achieved with current features? We've designed Data Designer to be composable – sometimes creative combinations of existing tools can accomplish what you need. Check out our examples or open an issue if you'd like help exploring this! +1. πŸ§— **Explore what's possible**: Can your use case be achieved with current features? We've designed Data Designer to be composable – sometimes creative combinations of existing tools can accomplish what you need. Check out our examples or open an issue if you'd like help exploring this! -2. πŸ”Œ **Extend with a plugin**: If existing features aren't quite enough, can your idea be implemented and shared as a plugin that extends the core library? +2. πŸ”Œ **Extend through plugins**: If existing features aren't quite enough, consider implementing your idea as a plugin that extends the core library. Plugins let you experiment and share innovations while keeping the core focused. -3. βš™οΈ **Integrate with the core library**: If your feature idea or existing plugin is broadly useful and aligns with Data Designer's goals, we'd love to integrate it! We're happy to discuss whether it is a good fit for the core library. +3. βš™οΈ **Integrate into the core**: If your feature or plugin proves broadly useful and aligns with Data Designer's goals, we'd love to integrate it into the core! We're happy to discuss whether it's a good fit and how to move forward together. + +This approach helps us grow thoughtfully while keeping Data Designer reliable for everyone. ### Submitting a Feature Request From e9f97d60add13a099fff801867f2a051b90654b7 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 31 Oct 2025 10:43:16 -0400 Subject: [PATCH 15/21] git branch pattern update --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 70ed22a12..3e0a48f4e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -141,7 +141,7 @@ Data Designer uses [`uv`](https://github.com/astral-sh/uv) for dependency manage ```bash git checkout main git pull upstream main - git checkout -b //#- + git checkout -b //- ``` Example types of change: @@ -155,7 +155,7 @@ Data Designer uses [`uv`](https://github.com/astral-sh/uv) for dependency manage - `perf` for performance improvements Example branch name: - - `feat/johnnygreco/#123-add-xyz-generator` for a new feature by @johnnygreco, addressing issue #123 + - `johnnygreco/feat/123-add-xyz-generator` for a new feature by @johnnygreco, addressing issue #123 2. **Develop your changes** @@ -207,7 +207,7 @@ Ensure your changes meet the following criteria: 1. **Push your changes** to your fork: ```bash - git push origin //#- + git push origin //- ``` 2. **Open a pull request** on GitHub from your fork to the main repository From 5765301ef9f432c73dac03c9c66e6da763a994e7 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 31 Oct 2025 13:18:34 -0400 Subject: [PATCH 16/21] Update CONTRIBUTING.md Co-authored-by: Nabin Mulepati --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e0a48f4e..7f2eb3f9f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -218,6 +218,7 @@ Ensure your changes meet the following criteria: - Maintainers will review your PR and may request changes - Address feedback by pushing additional commits to your branch +- Reply to the feedback comment with a link to the commit that addresses it. - Once approved, a maintainer will merge your PR - Your contribution will be included in the next release! From 2c38781909e1486c21744075d70f950ed3e0fe66 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 31 Oct 2025 13:21:09 -0400 Subject: [PATCH 17/21] agent md blurb --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f2eb3f9f..4c22e624f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,7 +25,7 @@ Whether you're new to the project or ready to dive in, the resources below will 1. **[README.md](README.md)** – best place to start to learn the basics of the project -2. **[AGENTS.md](AGENTS.md)** – code structure and development guidelines for humans and agents +2. **[AGENTS.md](AGENTS.md)** – context and instructions to help AI coding agents work on Data Designer (it's also useful for human developers!) 3. **[Documentation](docs/)** – detailed documentation on Data Designer's capabilities and usage From f405dbd3ba298f5cd0ee002bbab09dd11a2b7c1c Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 31 Oct 2025 13:26:07 -0400 Subject: [PATCH 18/21] pr feedback --- CONTRIBUTING.md | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c22e624f..fc7a6b61e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,11 +61,9 @@ Share your use cases! Example notebooks and tutorials help others understand how Help us improve test coverage by adding tests for untested code paths or edge cases. ## Feature Requests - Data Designer is designed to be as flexible and extensible as possible, and we welcome your ideas for pushing its capabilities even further! To keep the core library maintainable, while also supporting innovation, we take an incremental approach: we explore what's already possible, extend through plugins when needed, and integrate the most broadly useful features into the core library: ### How We Grow Data Designer - 1. πŸ§— **Explore what's possible**: Can your use case be achieved with current features? We've designed Data Designer to be composable – sometimes creative combinations of existing tools can accomplish what you need. Check out our examples or open an issue if you'd like help exploring this! 2. πŸ”Œ **Extend through plugins**: If existing features aren't quite enough, consider implementing your idea as a plugin that extends the core library. Plugins let you experiment and share innovations while keeping the core focused. @@ -75,7 +73,6 @@ Data Designer is designed to be as flexible and extensible as possible, and we w This approach helps us grow thoughtfully while keeping Data Designer reliable for everyone. ### Submitting a Feature Request - Open a [new issue](https://github.com/NVIDIA-NeMo/DataDesigner/issues/new) with: - **Clear title**: Concise description of the feature @@ -86,11 +83,9 @@ Open a [new issue](https://github.com/NVIDIA-NeMo/DataDesigner/issues/new) with: - **Willingness to implement**: Are you interested in implementing this yourself? ## Development Guide - Data Designer uses [`uv`](https://github.com/astral-sh/uv) for dependency management. If you don't have uv installed, follow their [installation instructions](https://docs.astral.sh/uv/getting-started/installation/). ### Initial Setup - 0. **Create or find an issue** Before starting work, ensure there's an issue tracking your contribution: @@ -120,12 +115,6 @@ Data Designer uses [`uv`](https://github.com/astral-sh/uv) for dependency manage make install-dev-notebooks ``` - This creates a virtual environment in `.venv`. Activate it with: - - ```bash - source .venv/bin/activate - ``` - 3. **Verify your setup** ```bash @@ -223,13 +212,11 @@ Ensure your changes meet the following criteria: - Your contribution will be included in the next release! ## Code of Conduct - Data Designer follows the Contributor Covenant Code of Conduct. We are committed to providing a welcoming and inclusive environment for all contributors. **Please read our complete [Code of Conduct](CODE_OF_CONDUCT.md)** for full details on our standards and expectations. -### Copyright - +### License File Headers All code files that are added to this repository must include the appropriate NVIDIA copyright header: ```python @@ -240,7 +227,6 @@ All code files that are added to this repository must include the appropriate NV Use `make update-license-headers` to add headers automatically. ## Getting Help - Need help with your contribution? - **Documentation**: Check the [documentation](docs/) and [AGENTS.md](AGENTS.md) for additional information From 13f9527f337712b5d40d6e952bd2a879f10ef8a2 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 3 Nov 2025 10:29:14 -0500 Subject: [PATCH 19/21] some rewording --- CONTRIBUTING.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc7a6b61e..90b6ce623 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,16 +61,16 @@ Share your use cases! Example notebooks and tutorials help others understand how Help us improve test coverage by adding tests for untested code paths or edge cases. ## Feature Requests -Data Designer is designed to be as flexible and extensible as possible, and we welcome your ideas for pushing its capabilities even further! To keep the core library maintainable, while also supporting innovation, we take an incremental approach: we explore what's already possible, extend through plugins when needed, and integrate the most broadly useful features into the core library: +Data Designer is designed to be as flexible and extensible as possible, and we welcome your ideas for pushing its capabilities even further! To keep the core library maintainable, while also supporting innovation, we take an incremental approach when adding new features: we explore what's already possible, extend through plugins when needed, and integrate the most broadly useful features into the core library: ### How We Grow Data Designer 1. πŸ§— **Explore what's possible**: Can your use case be achieved with current features? We've designed Data Designer to be composable – sometimes creative combinations of existing tools can accomplish what you need. Check out our examples or open an issue if you'd like help exploring this! -2. πŸ”Œ **Extend through plugins**: If existing features aren't quite enough, consider implementing your idea as a plugin that extends the core library. Plugins let you experiment and share innovations while keeping the core focused. +2. πŸ”Œ **Extend through plugins**: If existing features aren't quite enough, consider implementing your idea as a plugin that extends the core library. Plugins let you experiment and share functionality while keeping the core library focused. -3. βš™οΈ **Integrate into the core**: If your feature or plugin proves broadly useful and aligns with Data Designer's goals, we'd love to integrate it into the core! We're happy to discuss whether it's a good fit and how to move forward together. +3. βš™οΈ **Integrate into the core library**: If your feature or plugin proves broadly useful and aligns with Data Designer's goals, we'd love to integrate it into the core library! We're happy to discuss whether it's a good fit and how to move forward together. -This approach helps us grow thoughtfully while keeping Data Designer reliable for everyone. +This approach helps us grow thoughtfully while keeping Data Designer focused and maintainable. ### Submitting a Feature Request Open a [new issue](https://github.com/NVIDIA-NeMo/DataDesigner/issues/new) with: From c84a70bca3f58067c4ce82fe5dd176f6c3a2fb15 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 3 Nov 2025 10:35:57 -0500 Subject: [PATCH 20/21] punctuation --- CONTRIBUTING.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 90b6ce623..3a95ba8aa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,7 +61,7 @@ Share your use cases! Example notebooks and tutorials help others understand how Help us improve test coverage by adding tests for untested code paths or edge cases. ## Feature Requests -Data Designer is designed to be as flexible and extensible as possible, and we welcome your ideas for pushing its capabilities even further! To keep the core library maintainable, while also supporting innovation, we take an incremental approach when adding new features: we explore what's already possible, extend through plugins when needed, and integrate the most broadly useful features into the core library: +Data Designer is designed to be as flexible and extensible as possible, and we welcome your ideas for pushing its capabilities even further! To keep the core library maintainable, while also supporting innovation, we take an incremental approach when adding new features – we explore what's already possible, extend through plugins when needed, and integrate the most broadly useful features into the core library: ### How We Grow Data Designer 1. πŸ§— **Explore what's possible**: Can your use case be achieved with current features? We've designed Data Designer to be composable – sometimes creative combinations of existing tools can accomplish what you need. Check out our examples or open an issue if you'd like help exploring this! @@ -148,7 +148,7 @@ Data Designer uses [`uv`](https://github.com/astral-sh/uv) for dependency manage 2. **Develop your changes** - Please follow the patterns and conventions outlined in [AGENTS.md](AGENTS.md). + Please follow the patterns and conventions used throughout the codebase, as well as those outlined in [AGENTS.md](AGENTS.md). 3. **Test and validate** @@ -183,7 +183,6 @@ Data Designer uses [`uv`](https://github.com/astral-sh/uv) for dependency manage Ensure your changes meet the following criteria: -- Code follows patterns and conventions outlined in [AGENTS.md](AGENTS.md) - All tests pass (`make test`) - Code is formatted and linted (`make check-all-fix`) - New functionality includes tests @@ -201,7 +200,7 @@ Ensure your changes meet the following criteria: 2. **Open a pull request** on GitHub from your fork to the main repository -3. **Respond to review feedback** promptly and update your PR as needed +3. **Respond to review feedback** update your PR as needed ### Pull Request Review Process From 17657e6a850572e10aae8969fa70f5f4fa58713c Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Mon, 3 Nov 2025 16:08:00 -0500 Subject: [PATCH 21/21] missing quote --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d51666f83..d941b66ea 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ help: @echo "" @echo "πŸ› οΈ Utilities:" @echo " clean - Remove coverage reports and cache files" - @echo " serve-docs-locally - Serve documentation locally + @echo " serve-docs-locally - Serve documentation locally" @echo " check-license-headers - Check if all files have license headers" @echo " update-license-headers - Add license headers to all files" @echo ""