From dbd594d7f10b08652ac4ce518f0f876015a72c84 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Wed, 29 Apr 2026 16:02:48 -0600 Subject: [PATCH 1/7] feat(config): add deterministic fingerprint for workflow configs (#584) Provides DataDesignerConfig.fingerprint() and a freestanding fingerprint_config() helper that produce a content-addressable sha256 hash of the data-relevant portion of a workflow config. Identical configs hash identically across processes and Python versions; fields that don't affect generated rows (tool_configs, profilers, skip_health_check, max_parallel_requests, timeout, HuggingFace seed token/endpoint) are excluded. Custom column generators contribute their registered name and generator_params (L1) by default; opt-in custom_column_source=True also hashes inspect.getsource() of each generator (L2) and degrades gracefully with a warning when the source can't be retrieved. The normalization scheme is versioned via CONFIG_HASH_VERSION so future changes can be detected as "unknown identity" rather than mismatch. --- .../src/data_designer/config/__init__.py | 9 + .../config/data_designer_config.py | 18 + .../src/data_designer/config/fingerprint.py | 187 ++++++++++ .../tests/config/test_fingerprint.py | 343 ++++++++++++++++++ 4 files changed, 557 insertions(+) create mode 100644 packages/data-designer-config/src/data_designer/config/fingerprint.py create mode 100644 packages/data-designer-config/tests/config/test_fingerprint.py diff --git a/packages/data-designer-config/src/data_designer/config/__init__.py b/packages/data-designer-config/src/data_designer/config/__init__.py index eb385e15a..80c9d8a54 100644 --- a/packages/data-designer-config/src/data_designer/config/__init__.py +++ b/packages/data-designer-config/src/data_designer/config/__init__.py @@ -32,6 +32,11 @@ from data_designer.config.config_builder import DataDesignerConfigBuilder # noqa: F401 from data_designer.config.custom_column import custom_column_generator # noqa: F401 from data_designer.config.data_designer_config import DataDesignerConfig # noqa: F401 + from data_designer.config.fingerprint import ( # noqa: F401 + CONFIG_HASH_ALGO, + CONFIG_HASH_VERSION, + fingerprint_config, + ) from data_designer.config.mcp import ( # noqa: F401 LocalStdioMCPProvider, MCPProvider, @@ -150,6 +155,10 @@ "custom_column_generator": (f"{_MOD_BASE}.custom_column", "custom_column_generator"), # data_designer_config "DataDesignerConfig": (f"{_MOD_BASE}.data_designer_config", "DataDesignerConfig"), + # fingerprint + "CONFIG_HASH_ALGO": (f"{_MOD_BASE}.fingerprint", "CONFIG_HASH_ALGO"), + "CONFIG_HASH_VERSION": (f"{_MOD_BASE}.fingerprint", "CONFIG_HASH_VERSION"), + "fingerprint_config": (f"{_MOD_BASE}.fingerprint", "fingerprint_config"), # mcp "LocalStdioMCPProvider": (_MOD_MCP, "LocalStdioMCPProvider"), "MCPProvider": (_MOD_MCP, "MCPProvider"), diff --git a/packages/data-designer-config/src/data_designer/config/data_designer_config.py b/packages/data-designer-config/src/data_designer/config/data_designer_config.py index 0fc2a96e5..d7e06b26b 100644 --- a/packages/data-designer-config/src/data_designer/config/data_designer_config.py +++ b/packages/data-designer-config/src/data_designer/config/data_designer_config.py @@ -10,6 +10,7 @@ from data_designer.config.analysis.column_profilers import ColumnProfilerConfigT from data_designer.config.column_types import ColumnConfigT from data_designer.config.exportable_config import ExportableConfigBase +from data_designer.config.fingerprint import fingerprint_config from data_designer.config.mcp import ToolConfig from data_designer.config.models import ModelConfig from data_designer.config.processor_types import ProcessorConfigT @@ -42,3 +43,20 @@ class DataDesignerConfig(ExportableConfigBase): constraints: list[ColumnConstraintInputT] | None = None profilers: list[ColumnProfilerConfigT] | None = None processors: list[Annotated[ProcessorConfigT, Field(discriminator="processor_type")]] | None = None + + def fingerprint(self, *, custom_column_source: bool = False) -> dict[str, str | int]: + """Compute a deterministic content-addressable fingerprint of this config. + + See :func:`data_designer.config.fingerprint.fingerprint_config` for the + full list of identity-relevant and excluded fields, and for the L1/L2 + custom-column behavior. + + Args: + custom_column_source: If True, additionally hash the source of each + custom column generator (L2). Defaults to False. + + Returns: + A dict with ``config_hash``, ``config_hash_algo``, and + ``config_hash_version``. + """ + return fingerprint_config(self, custom_column_source=custom_column_source) diff --git a/packages/data-designer-config/src/data_designer/config/fingerprint.py b/packages/data-designer-config/src/data_designer/config/fingerprint.py new file mode 100644 index 000000000..c84c34eed --- /dev/null +++ b/packages/data-designer-config/src/data_designer/config/fingerprint.py @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Deterministic content-addressable fingerprint for a workflow config. + +The fingerprint identifies the *data-relevant* portion of a +:class:`DataDesignerConfig` so that two configs producing the same dataset hash +to the same value, while configs differing only in environment, runtime, or +post-generation analysis hash to different values when they should and to the +same value when they shouldn't. + +The hash is computed over a canonical JSON dump of the config (Pydantic +``model_dump(mode="json")``) with non-identity fields removed. Dict keys are +sorted, list order is preserved (list order is part of identity). + +The normalization scheme is versioned via :data:`CONFIG_HASH_VERSION`. Persist +the version alongside the hash so future scheme changes can be detected as +"unknown identity" rather than "definite mismatch". +""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import logging +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +from data_designer.config.column_configs import CustomColumnConfig + +if TYPE_CHECKING: + from data_designer.config.data_designer_config import DataDesignerConfig + +logger = logging.getLogger(__name__) + +CONFIG_HASH_VERSION = 1 +CONFIG_HASH_ALGO = "sha256" + +# Top-level DataDesignerConfig keys excluded from the fingerprint. +# tool_configs -- MCP tool wiring is a runtime/execution choice +# profilers -- post-generation analysis, doesn't affect generated rows +_EXCLUDED_TOP_LEVEL_KEYS: frozenset[str] = frozenset({"tool_configs", "profilers"}) + +# ModelConfig keys excluded -- env/runtime knobs. +_EXCLUDED_MODEL_KEYS: frozenset[str] = frozenset({"skip_health_check"}) + +# Inference-parameter keys excluded -- concurrency / timing only. +_EXCLUDED_INFERENCE_KEYS: frozenset[str] = frozenset({"max_parallel_requests", "timeout"}) + +# HuggingFaceSeedSource keys excluded -- auth and endpoint URL are not data identity. +_EXCLUDED_HF_SEED_KEYS: frozenset[str] = frozenset({"token", "endpoint"}) + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def fingerprint_config( + config: DataDesignerConfig, + *, + custom_column_source: bool = False, +) -> dict[str, str | int]: + """Compute a deterministic fingerprint of a workflow config. + + The fingerprint is content-addressable: identical configs (modulo excluded + fields) produce identical hashes across processes, Python versions, and + module load orders. Changing any identity-relevant field changes the hash; + changing an excluded field does not. + + Identity-relevant fields: + * ``columns`` (names, types, generator params, processors, validators, + skip/drop/propagate_skip flags) + * ``model_configs`` (alias, model, provider, sampling-relevant inference + params -- temperature, top_p, max_tokens, extra_body) + * ``seed_config`` (source path / sampling strategy / selection strategy) + * ``constraints`` + * top-level ``processors`` + + Excluded fields: + * ``tool_configs`` (runtime tool wiring) + * ``profilers`` (post-generation analysis) + * ``model_configs[*].skip_health_check`` + * ``inference_parameters.max_parallel_requests``, ``inference_parameters.timeout`` + * HuggingFace seed source ``token`` and ``endpoint`` + + Custom column generators are always identified by registered function name + and ``generator_params`` (L1). When ``custom_column_source=True``, the + function source is also hashed (L2); plugins whose source cannot be + retrieved degrade gracefully with a warning. + + Note: ``buffer_size`` lives on :class:`RunConfig`, not on + :class:`DataDesignerConfig`, and is therefore not part of this fingerprint. + The fingerprint identifies the workflow definition; runtime knobs that + don't change the final dataset are out of scope. + + Args: + config: The workflow config to fingerprint. + custom_column_source: If True, also hash ``inspect.getsource()`` of + each custom column generator (L2). Defaults to False (L1 only). + + Returns: + A dict with ``config_hash`` (``"sha256:..."``), ``config_hash_algo``, + and ``config_hash_version`` suitable for embedding in dataset metadata. + """ + payload: dict[str, Any] = {"config": _normalize_config_dict(config.to_dict())} + if custom_column_source: + payload["custom_column_sources"] = _collect_custom_column_sources(config) + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return { + "config_hash": f"{CONFIG_HASH_ALGO}:{digest}", + "config_hash_algo": CONFIG_HASH_ALGO, + "config_hash_version": CONFIG_HASH_VERSION, + } + + +# --------------------------------------------------------------------------- +# Private helpers +# --------------------------------------------------------------------------- + + +def _drop_keys(source: dict[str, Any], keys: frozenset[str]) -> dict[str, Any]: + return {k: v for k, v in source.items() if k not in keys} + + +def _normalize_model_config(model_config: dict[str, Any]) -> dict[str, Any]: + normalized = _drop_keys(model_config, _EXCLUDED_MODEL_KEYS) + inference_params = normalized.get("inference_parameters") + if isinstance(inference_params, dict): + normalized["inference_parameters"] = _drop_keys(inference_params, _EXCLUDED_INFERENCE_KEYS) + return normalized + + +def _normalize_seed_config(seed_config: dict[str, Any]) -> dict[str, Any]: + normalized = dict(seed_config) + seed_source = normalized.get("source") + if isinstance(seed_source, dict) and seed_source.get("seed_type") == "hf": + normalized["source"] = _drop_keys(seed_source, _EXCLUDED_HF_SEED_KEYS) + return normalized + + +def _normalize_config_dict(config_dict: dict[str, Any]) -> dict[str, Any]: + normalized = _drop_keys(config_dict, _EXCLUDED_TOP_LEVEL_KEYS) + model_configs = normalized.get("model_configs") + if model_configs: + normalized["model_configs"] = [_normalize_model_config(mc) for mc in model_configs] + seed_config = normalized.get("seed_config") + if seed_config: + normalized["seed_config"] = _normalize_seed_config(seed_config) + return normalized + + +def _hash_custom_column_source(fn: Callable[..., Any], column_name: str) -> str | None: + """Hash the source of a custom column generator (L2). + + Returns the sha256 hex digest of the function source, or ``None`` if the + source cannot be retrieved (e.g., compiled / zipped plugin, C extension, + interactively-defined function). Plugins that can't be source-hashed + degrade gracefully with a warning rather than raising. + """ + try: + unwrapped = inspect.unwrap(fn) + source = inspect.getsource(unwrapped) + except (OSError, TypeError) as exc: + logger.warning( + "Could not retrieve source for custom column %r generator (%s); " + "fingerprint will not detect implementation changes for this column.", + column_name, + exc, + ) + return None + return hashlib.sha256(source.encode("utf-8")).hexdigest() + + +def _collect_custom_column_sources(config: DataDesignerConfig) -> list[dict[str, Any]]: + sources: list[dict[str, Any]] = [] + for col in config.columns: + if isinstance(col, CustomColumnConfig): + sources.append( + { + "name": col.name, + "source_hash": _hash_custom_column_source(col.generator_function, col.name), + } + ) + return sources diff --git a/packages/data-designer-config/tests/config/test_fingerprint.py b/packages/data-designer-config/tests/config/test_fingerprint.py new file mode 100644 index 000000000..89c88172e --- /dev/null +++ b/packages/data-designer-config/tests/config/test_fingerprint.py @@ -0,0 +1,343 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import inspect +import subprocess +import sys +from collections.abc import Callable +from typing import Any + +import pytest +import yaml +from pydantic import BaseModel + +from data_designer.config import fingerprint as fp_mod +from data_designer.config.analysis.column_profilers import JudgeScoreProfilerConfig +from data_designer.config.base import SkipConfig +from data_designer.config.column_configs import ( + CustomColumnConfig, + LLMTextColumnConfig, + SamplerColumnConfig, +) +from data_designer.config.custom_column import custom_column_generator +from data_designer.config.data_designer_config import DataDesignerConfig +from data_designer.config.fingerprint import ( + CONFIG_HASH_ALGO, + CONFIG_HASH_VERSION, + fingerprint_config, +) +from data_designer.config.mcp import ToolConfig +from data_designer.config.models import ChatCompletionInferenceParams, ModelConfig +from data_designer.config.sampler_params import CategorySamplerParams, UniformSamplerParams +from data_designer.config.seed import SeedConfig +from data_designer.config.seed_source import HuggingFaceSeedSource + + +def _hash(config: DataDesignerConfig, *, custom_column_source: bool = False) -> str: + return str(fingerprint_config(config, custom_column_source=custom_column_source)["config_hash"]) + + +def test_fingerprint_shape(stub_data_designer_config: DataDesignerConfig) -> None: + fp = stub_data_designer_config.fingerprint() + assert set(fp.keys()) == {"config_hash", "config_hash_algo", "config_hash_version"} + assert fp["config_hash_algo"] == CONFIG_HASH_ALGO + assert fp["config_hash_version"] == CONFIG_HASH_VERSION + assert fp["config_hash"].startswith(f"{CONFIG_HASH_ALGO}:") + digest = fp["config_hash"].split(":", 1)[1] + assert len(digest) == 64 # sha256 hex + assert all(c in "0123456789abcdef" for c in digest) + + +def test_fingerprint_deterministic_within_process( + stub_data_designer_config: DataDesignerConfig, + stub_data_designer_config_str: str, +) -> None: + rebuilt = DataDesignerConfig.model_validate(yaml.safe_load(stub_data_designer_config_str)) + assert _hash(stub_data_designer_config) == _hash(rebuilt) + + +def test_fingerprint_deterministic_across_processes(stub_data_designer_config_str: str) -> None: + """A separate Python process must produce the same digest for the same config.""" + script = f""" +import sys, yaml +from data_designer.config.data_designer_config import DataDesignerConfig +from data_designer.config.fingerprint import fingerprint_config + +cfg = DataDesignerConfig.model_validate(yaml.safe_load({stub_data_designer_config_str!r})) +sys.stdout.write(fingerprint_config(cfg)["config_hash"]) +""" + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, check=True) + out = result.stdout.strip() + + cfg = DataDesignerConfig.model_validate(yaml.safe_load(stub_data_designer_config_str)) + assert out == _hash(cfg) + + +# --------------------------------------------------------------------------- +# Helpers for building minimal configs in include/exclude tests. +# --------------------------------------------------------------------------- + + +def _make_model() -> ModelConfig: + return ModelConfig( + alias="m", + model="some-model", + inference_parameters=ChatCompletionInferenceParams(temperature=0.5, top_p=0.9, max_tokens=128), + ) + + +def _make_minimal_config(**overrides: object) -> DataDesignerConfig: + base: dict[str, Any] = { + "columns": [SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1))], + "model_configs": [_make_model()], + } + base.update(overrides) + return DataDesignerConfig(**base) + + +# --------------------------------------------------------------------------- +# INCLUDE: identity-relevant changes must change the hash. +# --------------------------------------------------------------------------- + + +def test_changing_column_name_changes_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config( + columns=[SamplerColumnConfig(name="y", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1))], + ) + assert _hash(a) != _hash(b) + + +def test_changing_column_type_changes_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config( + columns=[ + SamplerColumnConfig(name="x", sampler_type="category", params=CategorySamplerParams(values=["a", "b"])), + ], + ) + assert _hash(a) != _hash(b) + + +def test_changing_sampler_params_changes_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config( + columns=[SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=2))], + ) + assert _hash(a) != _hash(b) + + +def test_changing_model_identity_changes_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config(model_configs=[ModelConfig(alias="m", model="other-model")]) + assert _hash(a) != _hash(b) + + +def test_changing_temperature_changes_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config( + model_configs=[ + ModelConfig( + alias="m", + model="some-model", + inference_parameters=ChatCompletionInferenceParams(temperature=0.99, top_p=0.9, max_tokens=128), + ) + ], + ) + assert _hash(a) != _hash(b) + + +def test_changing_column_order_changes_hash() -> None: + cols_a = [ + SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), + SamplerColumnConfig(name="y", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), + ] + cols_b = list(reversed(cols_a)) + assert _hash(_make_minimal_config(columns=cols_a)) != _hash(_make_minimal_config(columns=cols_b)) + + +def test_changing_skip_changes_hash() -> None: + base_col = LLMTextColumnConfig(name="t", prompt="hi {{x}}", model_alias="m") + skipped = LLMTextColumnConfig( + name="t", + prompt="hi {{x}}", + model_alias="m", + skip=SkipConfig(when="{{ x > 0 }}"), + ) + cols_no_skip = [ + SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), + base_col, + ] + cols_skip = [ + SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), + skipped, + ] + assert _hash(_make_minimal_config(columns=cols_no_skip)) != _hash(_make_minimal_config(columns=cols_skip)) + + +# --------------------------------------------------------------------------- +# EXCLUDE: non-identity changes must NOT change the hash. +# --------------------------------------------------------------------------- + + +def test_skip_health_check_does_not_change_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config( + model_configs=[ + ModelConfig( + alias="m", + model="some-model", + inference_parameters=ChatCompletionInferenceParams(temperature=0.5, top_p=0.9, max_tokens=128), + skip_health_check=True, + ) + ], + ) + assert _hash(a) == _hash(b) + + +def test_max_parallel_requests_does_not_change_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config( + model_configs=[ + ModelConfig( + alias="m", + model="some-model", + inference_parameters=ChatCompletionInferenceParams( + temperature=0.5, top_p=0.9, max_tokens=128, max_parallel_requests=32 + ), + ) + ], + ) + assert _hash(a) == _hash(b) + + +def test_inference_timeout_does_not_change_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config( + model_configs=[ + ModelConfig( + alias="m", + model="some-model", + inference_parameters=ChatCompletionInferenceParams( + temperature=0.5, top_p=0.9, max_tokens=128, timeout=30 + ), + ) + ], + ) + assert _hash(a) == _hash(b) + + +def test_tool_configs_do_not_change_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"])]) + assert _hash(a) == _hash(b) + + +def test_profilers_do_not_change_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config(profilers=[JudgeScoreProfilerConfig(model_alias="m")]) + assert _hash(a) == _hash(b) + + +def test_hf_seed_token_and_endpoint_do_not_change_hash() -> None: + a = _make_minimal_config( + seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/data.csv")), + ) + b = _make_minimal_config( + seed_config=SeedConfig( + source=HuggingFaceSeedSource( + path="datasets/x/y/data.csv", + token="secret", + endpoint="https://example.com", + ), + ), + ) + assert _hash(a) == _hash(b) + + +def test_changing_hf_seed_path_changes_hash() -> None: + a = _make_minimal_config(seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/a.csv"))) + b = _make_minimal_config(seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/b.csv"))) + assert _hash(a) != _hash(b) + + +# --------------------------------------------------------------------------- +# Custom columns: L1 (default) and L2 (opt-in source hashing). +# --------------------------------------------------------------------------- + + +class _GenParamsV1(BaseModel): + factor: int = 1 + + +@custom_column_generator() +def _generate_v1(row: dict, generator_params: _GenParamsV1) -> str: # pragma: no cover - logic not exercised + return str(row.get("x", 0) * generator_params.factor) + + +@custom_column_generator() +def _generate_v2(row: dict, generator_params: _GenParamsV1) -> str: # pragma: no cover - logic not exercised + return str(row.get("x", 0) * generator_params.factor + 1) + + +def _make_custom_config(fn: Callable[..., Any], params: _GenParamsV1 | None = None) -> DataDesignerConfig: + return _make_minimal_config( + columns=[ + SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), + CustomColumnConfig( + name="c", + generator_function=fn, + generator_params=params or _GenParamsV1(), + ), + ], + ) + + +def test_custom_column_l1_includes_generator_params() -> None: + a = _make_custom_config(_generate_v1, _GenParamsV1(factor=1)) + b = _make_custom_config(_generate_v1, _GenParamsV1(factor=2)) + assert _hash(a) != _hash(b) + + +def test_custom_column_l1_includes_generator_function_name() -> None: + a = _make_custom_config(_generate_v1) + b = _make_custom_config(_generate_v2) + # Different function names serialize to different values via field_serializer. + assert _hash(a) != _hash(b) + + +def test_custom_column_l2_detects_source_change(monkeypatch: pytest.MonkeyPatch) -> None: + a = _make_custom_config(_generate_v1) + base = _hash(a, custom_column_source=True) + + # Simulate an implementation edit by feeding a different source string. + sources = iter(["original-source", "edited-source"]) + monkeypatch.setattr(fp_mod, "_hash_custom_column_source", lambda fn, name: next(sources)) + + edit_first = _hash(a, custom_column_source=True) + edit_second = _hash(a, custom_column_source=True) + assert edit_first != edit_second + assert base != edit_first + + +def test_custom_column_unhashable_source_degrades_gracefully( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A plugin whose source can't be retrieved should warn, not raise.""" + + def _raise_oserror(_fn: object) -> str: + raise OSError("compiled / zipped plugin") + + monkeypatch.setattr(inspect, "getsource", _raise_oserror) + + a = _make_custom_config(_generate_v1) + with caplog.at_level("WARNING", logger=fp_mod.__name__): + out = a.fingerprint(custom_column_source=True) + assert out["config_hash"].startswith(f"{CONFIG_HASH_ALGO}:") + assert "Could not retrieve source" in caplog.text + + +def test_l1_and_l2_produce_different_hashes() -> None: + a = _make_custom_config(_generate_v1) + assert _hash(a) != _hash(a, custom_column_source=True) From 34748f17342cf7c5e58ee516437c9d684519b0de Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Wed, 29 Apr 2026 19:21:51 -0600 Subject: [PATCH 2/7] test(config): cover constraints, processors, extra_body, provider, and seed strategies in fingerprint tests Also document L1 __name__-collision and L2 whitespace-sensitivity limitations in fingerprint_config(), and drop the json.dumps default=str fallback so non-JSON-native values fail loudly instead of silently degrading determinism. --- .../src/data_designer/config/fingerprint.py | 13 +++- .../tests/config/test_fingerprint.py | 78 ++++++++++++++++++- 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/packages/data-designer-config/src/data_designer/config/fingerprint.py b/packages/data-designer-config/src/data_designer/config/fingerprint.py index c84c34eed..1ae87a0ad 100644 --- a/packages/data-designer-config/src/data_designer/config/fingerprint.py +++ b/packages/data-designer-config/src/data_designer/config/fingerprint.py @@ -95,6 +95,16 @@ def fingerprint_config( The fingerprint identifies the workflow definition; runtime knobs that don't change the final dataset are out of scope. + Limitations: + * **L1 collisions on ``__name__``**: custom columns are identified at L1 + by the generator's bare ``__name__``, not its qualified module path. + Two unrelated generators in different modules with the same name and + identical ``generator_params`` will produce the same L1 hash. Pass + ``custom_column_source=True`` to disambiguate via source. + * **L2 hashes raw source**: comment-only and formatting changes to a + generator's source will change the L2 hash even though they don't + affect behavior. + Args: config: The workflow config to fingerprint. custom_column_source: If True, also hash ``inspect.getsource()`` of @@ -107,7 +117,8 @@ def fingerprint_config( payload: dict[str, Any] = {"config": _normalize_config_dict(config.to_dict())} if custom_column_source: payload["custom_column_sources"] = _collect_custom_column_sources(config) - canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + # No `default=` fallback: a non-JSON-native value would silently break determinism (e.g., repr with memory addresses). + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() return { "config_hash": f"{CONFIG_HASH_ALGO}:{digest}", diff --git a/packages/data-designer-config/tests/config/test_fingerprint.py b/packages/data-designer-config/tests/config/test_fingerprint.py index 89c88172e..3ac878c1c 100644 --- a/packages/data-designer-config/tests/config/test_fingerprint.py +++ b/packages/data-designer-config/tests/config/test_fingerprint.py @@ -30,8 +30,10 @@ ) from data_designer.config.mcp import ToolConfig from data_designer.config.models import ChatCompletionInferenceParams, ModelConfig +from data_designer.config.processors import DropColumnsProcessorConfig +from data_designer.config.sampler_constraints import InequalityOperator, ScalarInequalityConstraint from data_designer.config.sampler_params import CategorySamplerParams, UniformSamplerParams -from data_designer.config.seed import SeedConfig +from data_designer.config.seed import IndexRange, SamplingStrategy, SeedConfig from data_designer.config.seed_source import HuggingFaceSeedSource @@ -176,6 +178,80 @@ def test_changing_skip_changes_hash() -> None: assert _hash(_make_minimal_config(columns=cols_no_skip)) != _hash(_make_minimal_config(columns=cols_skip)) +def test_changing_constraint_changes_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config( + constraints=[ScalarInequalityConstraint(target_column="x", operator=InequalityOperator.LT, rhs=0.5)], + ) + assert _hash(a) != _hash(b) + + +def test_changing_top_level_processor_changes_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config(processors=[DropColumnsProcessorConfig(name="drop", column_names=["x"])]) + assert _hash(a) != _hash(b) + + +def test_changing_extra_body_changes_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config( + model_configs=[ + ModelConfig( + alias="m", + model="some-model", + inference_parameters=ChatCompletionInferenceParams( + temperature=0.5, top_p=0.9, max_tokens=128, extra_body={"frequency_penalty": 0.5} + ), + ) + ], + ) + assert _hash(a) != _hash(b) + + +def test_changing_provider_changes_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config( + model_configs=[ + ModelConfig( + alias="m", + model="some-model", + provider="custom-provider", + inference_parameters=ChatCompletionInferenceParams(temperature=0.5, top_p=0.9, max_tokens=128), + ) + ], + ) + assert _hash(a) != _hash(b) + + +def test_changing_sampling_strategy_changes_hash() -> None: + a = _make_minimal_config( + seed_config=SeedConfig( + source=HuggingFaceSeedSource(path="datasets/x/y/data.csv"), + sampling_strategy=SamplingStrategy.ORDERED, + ), + ) + b = _make_minimal_config( + seed_config=SeedConfig( + source=HuggingFaceSeedSource(path="datasets/x/y/data.csv"), + sampling_strategy=SamplingStrategy.SHUFFLE, + ), + ) + assert _hash(a) != _hash(b) + + +def test_changing_selection_strategy_changes_hash() -> None: + a = _make_minimal_config( + seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/data.csv")), + ) + b = _make_minimal_config( + seed_config=SeedConfig( + source=HuggingFaceSeedSource(path="datasets/x/y/data.csv"), + selection_strategy=IndexRange(start=0, end=99), + ), + ) + assert _hash(a) != _hash(b) + + # --------------------------------------------------------------------------- # EXCLUDE: non-identity changes must NOT change the hash. # --------------------------------------------------------------------------- From a4f3bc6c95934a07f0e8fb2cb815c713468911d6 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Thu, 30 Apr 2026 11:39:04 -0600 Subject: [PATCH 3/7] feat(config): include tool_configs in fingerprint identity The set of MCP tools an LLM column can call (providers, allow_tools, max_tool_call_turns, tool_alias) shapes what the model produces, so tool_configs is identity-relevant. Only timeout_sec is excluded, mirroring how inference_parameters.timeout is treated as a runtime knob rather than a data-identity field. Updates the fingerprint_config docstring's Include/Exclude lists, flips the existing tool_configs exclusion test, and adds coverage for tool_alias / providers / allow_tools / max_tool_call_turns inclusion plus timeout_sec exclusion. Signed-off-by: Nabin Mulepati Made-with: Cursor --- .../src/data_designer/config/fingerprint.py | 17 +++++-- .../tests/config/test_fingerprint.py | 46 +++++++++++++++++-- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/packages/data-designer-config/src/data_designer/config/fingerprint.py b/packages/data-designer-config/src/data_designer/config/fingerprint.py index 1ae87a0ad..cb08fb8b4 100644 --- a/packages/data-designer-config/src/data_designer/config/fingerprint.py +++ b/packages/data-designer-config/src/data_designer/config/fingerprint.py @@ -38,9 +38,8 @@ CONFIG_HASH_ALGO = "sha256" # Top-level DataDesignerConfig keys excluded from the fingerprint. -# tool_configs -- MCP tool wiring is a runtime/execution choice # profilers -- post-generation analysis, doesn't affect generated rows -_EXCLUDED_TOP_LEVEL_KEYS: frozenset[str] = frozenset({"tool_configs", "profilers"}) +_EXCLUDED_TOP_LEVEL_KEYS: frozenset[str] = frozenset({"profilers"}) # ModelConfig keys excluded -- env/runtime knobs. _EXCLUDED_MODEL_KEYS: frozenset[str] = frozenset({"skip_health_check"}) @@ -48,6 +47,9 @@ # Inference-parameter keys excluded -- concurrency / timing only. _EXCLUDED_INFERENCE_KEYS: frozenset[str] = frozenset({"max_parallel_requests", "timeout"}) +# ToolConfig keys excluded -- per-call timing knob, analogous to inference_parameters.timeout. +_EXCLUDED_TOOL_CONFIG_KEYS: frozenset[str] = frozenset({"timeout_sec"}) + # HuggingFaceSeedSource keys excluded -- auth and endpoint URL are not data identity. _EXCLUDED_HF_SEED_KEYS: frozenset[str] = frozenset({"token", "endpoint"}) @@ -74,15 +76,17 @@ def fingerprint_config( skip/drop/propagate_skip flags) * ``model_configs`` (alias, model, provider, sampling-relevant inference params -- temperature, top_p, max_tokens, extra_body) + * ``tool_configs`` (alias, providers, allow_tools, max_tool_call_turns): + the set of MCP tools an LLM can call shapes what it produces * ``seed_config`` (source path / sampling strategy / selection strategy) * ``constraints`` * top-level ``processors`` Excluded fields: - * ``tool_configs`` (runtime tool wiring) * ``profilers`` (post-generation analysis) * ``model_configs[*].skip_health_check`` * ``inference_parameters.max_parallel_requests``, ``inference_parameters.timeout`` + * ``tool_configs[*].timeout_sec`` (per-call timing knob, not output identity) * HuggingFace seed source ``token`` and ``endpoint`` Custom column generators are always identified by registered function name @@ -152,11 +156,18 @@ def _normalize_seed_config(seed_config: dict[str, Any]) -> dict[str, Any]: return normalized +def _normalize_tool_config(tool_config: dict[str, Any]) -> dict[str, Any]: + return _drop_keys(tool_config, _EXCLUDED_TOOL_CONFIG_KEYS) + + def _normalize_config_dict(config_dict: dict[str, Any]) -> dict[str, Any]: normalized = _drop_keys(config_dict, _EXCLUDED_TOP_LEVEL_KEYS) model_configs = normalized.get("model_configs") if model_configs: normalized["model_configs"] = [_normalize_model_config(mc) for mc in model_configs] + tool_configs = normalized.get("tool_configs") + if tool_configs: + normalized["tool_configs"] = [_normalize_tool_config(tc) for tc in tool_configs] seed_config = normalized.get("seed_config") if seed_config: normalized["seed_config"] = _normalize_seed_config(seed_config) diff --git a/packages/data-designer-config/tests/config/test_fingerprint.py b/packages/data-designer-config/tests/config/test_fingerprint.py index 3ac878c1c..0e5f4dc48 100644 --- a/packages/data-designer-config/tests/config/test_fingerprint.py +++ b/packages/data-designer-config/tests/config/test_fingerprint.py @@ -252,6 +252,44 @@ def test_changing_selection_strategy_changes_hash() -> None: assert _hash(a) != _hash(b) +def test_adding_tool_config_changes_hash() -> None: + a = _make_minimal_config() + b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"])]) + assert _hash(a) != _hash(b) + + +def test_changing_tool_config_alias_changes_hash() -> None: + a = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t1", providers=["p"])]) + b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t2", providers=["p"])]) + assert _hash(a) != _hash(b) + + +def test_changing_tool_config_providers_changes_hash() -> None: + a = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p1"])]) + b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p2"])]) + assert _hash(a) != _hash(b) + + +def test_changing_tool_config_allow_tools_changes_hash() -> None: + a = _make_minimal_config( + tool_configs=[ToolConfig(tool_alias="t", providers=["p"], allow_tools=["search"])], + ) + b = _make_minimal_config( + tool_configs=[ToolConfig(tool_alias="t", providers=["p"], allow_tools=["search", "list"])], + ) + assert _hash(a) != _hash(b) + + +def test_changing_max_tool_call_turns_changes_hash() -> None: + a = _make_minimal_config( + tool_configs=[ToolConfig(tool_alias="t", providers=["p"], max_tool_call_turns=5)], + ) + b = _make_minimal_config( + tool_configs=[ToolConfig(tool_alias="t", providers=["p"], max_tool_call_turns=10)], + ) + assert _hash(a) != _hash(b) + + # --------------------------------------------------------------------------- # EXCLUDE: non-identity changes must NOT change the hash. # --------------------------------------------------------------------------- @@ -304,9 +342,11 @@ def test_inference_timeout_does_not_change_hash() -> None: assert _hash(a) == _hash(b) -def test_tool_configs_do_not_change_hash() -> None: - a = _make_minimal_config() - b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"])]) +def test_tool_config_timeout_sec_does_not_change_hash() -> None: + a = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"])]) + b = _make_minimal_config( + tool_configs=[ToolConfig(tool_alias="t", providers=["p"], timeout_sec=30.0)], + ) assert _hash(a) == _hash(b) From 2d590c179f164de082e798dd7471d690e5af870f Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Thu, 30 Apr 2026 11:57:29 -0600 Subject: [PATCH 4/7] no need to export config hash stuff to config --- .../src/data_designer/config/__init__.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/data-designer-config/src/data_designer/config/__init__.py b/packages/data-designer-config/src/data_designer/config/__init__.py index 80c9d8a54..eb385e15a 100644 --- a/packages/data-designer-config/src/data_designer/config/__init__.py +++ b/packages/data-designer-config/src/data_designer/config/__init__.py @@ -32,11 +32,6 @@ from data_designer.config.config_builder import DataDesignerConfigBuilder # noqa: F401 from data_designer.config.custom_column import custom_column_generator # noqa: F401 from data_designer.config.data_designer_config import DataDesignerConfig # noqa: F401 - from data_designer.config.fingerprint import ( # noqa: F401 - CONFIG_HASH_ALGO, - CONFIG_HASH_VERSION, - fingerprint_config, - ) from data_designer.config.mcp import ( # noqa: F401 LocalStdioMCPProvider, MCPProvider, @@ -155,10 +150,6 @@ "custom_column_generator": (f"{_MOD_BASE}.custom_column", "custom_column_generator"), # data_designer_config "DataDesignerConfig": (f"{_MOD_BASE}.data_designer_config", "DataDesignerConfig"), - # fingerprint - "CONFIG_HASH_ALGO": (f"{_MOD_BASE}.fingerprint", "CONFIG_HASH_ALGO"), - "CONFIG_HASH_VERSION": (f"{_MOD_BASE}.fingerprint", "CONFIG_HASH_VERSION"), - "fingerprint_config": (f"{_MOD_BASE}.fingerprint", "fingerprint_config"), # mcp "LocalStdioMCPProvider": (_MOD_MCP, "LocalStdioMCPProvider"), "MCPProvider": (_MOD_MCP, "MCPProvider"), From 28ed8e69818f1e67dca14af37fd9c0816ee262c2 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Thu, 30 Apr 2026 12:26:41 -0600 Subject: [PATCH 5/7] refactor(config): tighten fingerprint identity, drop L2 source hashing Drops the opt-in `custom_column_source` (L2) source-hashing path and addresses the canonicalization gaps the reviewers found. L2 had several silent footguns: closures with different captured state collapsed to the same hash, the empty `custom_column_sources: []` payload key made L1 and L2 disagree even on configs with no custom columns, `inspect.unwrap()` could raise `ValueError` on `__wrapped__` cycles (uncaught), and same-`__name__` collisions silently came back when `getsource()` failed. Removing it shrinks the public surface, deletes ~50 lines of helper code, and resolves seven review comments at once. Strengthens L1 identity for custom columns: the payload now includes `__qualname__`, `__module__`, and the `@custom_column_generator()` decorator metadata (`required_columns`, `side_effect_columns`, `model_aliases`) in addition to `__name__` + `generator_params`. This disambiguates same-`__name__`-different-scope collisions and prevents silently dropping DAG-affecting metadata. Canonicalizes alias-keyed lookup tables and optional collections so builder-API and YAML-loaded configs producing identical datasets fingerprint identically: * `model_configs` and `tool_configs` are sorted by alias before hashing (column order remains identity, since columns are DAG nodes). * `None` and `[]` collapse to "absent" for top-level optional collections (`model_configs`, `tool_configs`, `constraints`, `processors`) and for `tool_configs[*].allow_tools`. Consolidates the excluded-fields constants behind a single canonical table comment and drops the Sphinx `:func:`/`:class:` roles in the docstrings to match the rest of the codebase. Test coverage adds order-independence tests for `model_configs` and `tool_configs`, parametrized `None`-vs-`[]` equivalence tests for all four optional top-level collections plus `allow_tools`, qualname-disambiguation, and decorator-metadata change detection. Signed-off-by: Nabin Mulepati Made-with: Cursor --- .../config/data_designer_config.py | 18 +- .../src/data_designer/config/fingerprint.py | 240 +++++++++--------- .../tests/config/test_fingerprint.py | 137 +++++++--- 3 files changed, 229 insertions(+), 166 deletions(-) diff --git a/packages/data-designer-config/src/data_designer/config/data_designer_config.py b/packages/data-designer-config/src/data_designer/config/data_designer_config.py index d7e06b26b..86381332d 100644 --- a/packages/data-designer-config/src/data_designer/config/data_designer_config.py +++ b/packages/data-designer-config/src/data_designer/config/data_designer_config.py @@ -44,19 +44,15 @@ class DataDesignerConfig(ExportableConfigBase): profilers: list[ColumnProfilerConfigT] | None = None processors: list[Annotated[ProcessorConfigT, Field(discriminator="processor_type")]] | None = None - def fingerprint(self, *, custom_column_source: bool = False) -> dict[str, str | int]: + def fingerprint(self) -> dict[str, str | int]: """Compute a deterministic content-addressable fingerprint of this config. - See :func:`data_designer.config.fingerprint.fingerprint_config` for the - full list of identity-relevant and excluded fields, and for the L1/L2 - custom-column behavior. - - Args: - custom_column_source: If True, additionally hash the source of each - custom column generator (L2). Defaults to False. + See `data_designer.config.fingerprint.fingerprint_config` for the full + list of identity-relevant and excluded fields, and how custom column + generators are identified. Returns: - A dict with ``config_hash``, ``config_hash_algo``, and - ``config_hash_version``. + A dict with `config_hash`, `config_hash_algo`, and + `config_hash_version`. """ - return fingerprint_config(self, custom_column_source=custom_column_source) + return fingerprint_config(self) diff --git a/packages/data-designer-config/src/data_designer/config/fingerprint.py b/packages/data-designer-config/src/data_designer/config/fingerprint.py index cb08fb8b4..13d72af98 100644 --- a/packages/data-designer-config/src/data_designer/config/fingerprint.py +++ b/packages/data-designer-config/src/data_designer/config/fingerprint.py @@ -3,28 +3,30 @@ """Deterministic content-addressable fingerprint for a workflow config. -The fingerprint identifies the *data-relevant* portion of a -:class:`DataDesignerConfig` so that two configs producing the same dataset hash -to the same value, while configs differing only in environment, runtime, or -post-generation analysis hash to different values when they should and to the -same value when they shouldn't. +The fingerprint identifies the *data-relevant* portion of a `DataDesignerConfig` +so that two configs producing the same dataset hash to the same value, while +configs differing only in environment, runtime, or post-generation analysis +hash to different values when they should and to the same value when they +shouldn't. The hash is computed over a canonical JSON dump of the config (Pydantic -``model_dump(mode="json")``) with non-identity fields removed. Dict keys are -sorted, list order is preserved (list order is part of identity). - -The normalization scheme is versioned via :data:`CONFIG_HASH_VERSION`. Persist -the version alongside the hash so future scheme changes can be detected as +`model_dump(mode="json")`) with non-identity fields removed. Column order is +part of identity (DAG ordering); alias-keyed lookup tables (`model_configs`, +`tool_configs`) are sorted by alias so their internal order is irrelevant. +Empty/`None` optional collections are canonicalized to a single representation +so that builder-API and YAML-loaded configs producing identical datasets +fingerprint identically. + +The normalization scheme is versioned via `CONFIG_HASH_VERSION`. Persist the +version alongside the hash so future scheme changes can be detected as "unknown identity" rather than "definite mismatch". """ from __future__ import annotations import hashlib -import inspect import json -import logging -from collections.abc import Callable +from collections.abc import Iterable from typing import TYPE_CHECKING, Any from data_designer.config.column_configs import CustomColumnConfig @@ -32,38 +34,43 @@ if TYPE_CHECKING: from data_designer.config.data_designer_config import DataDesignerConfig -logger = logging.getLogger(__name__) - CONFIG_HASH_VERSION = 1 CONFIG_HASH_ALGO = "sha256" -# Top-level DataDesignerConfig keys excluded from the fingerprint. -# profilers -- post-generation analysis, doesn't affect generated rows -_EXCLUDED_TOP_LEVEL_KEYS: frozenset[str] = frozenset({"profilers"}) -# ModelConfig keys excluded -- env/runtime knobs. +# --------------------------------------------------------------------------- +# Excluded fields (single canonical table). Each entry is excluded from the +# fingerprint because it doesn't affect generated rows: +# +# profilers : post-generation analysis +# model_configs[*].skip_health_check : startup probe, not generation +# inference_parameters.{max_parallel_requests, timeout} +# : concurrency / timing only +# tool_configs[*].timeout_sec : per-call timing knob +# HuggingFaceSeedSource.{token, endpoint} +# : auth + env, not data identity +# --------------------------------------------------------------------------- +_EXCLUDED_TOP_LEVEL_KEYS: frozenset[str] = frozenset({"profilers"}) _EXCLUDED_MODEL_KEYS: frozenset[str] = frozenset({"skip_health_check"}) - -# Inference-parameter keys excluded -- concurrency / timing only. _EXCLUDED_INFERENCE_KEYS: frozenset[str] = frozenset({"max_parallel_requests", "timeout"}) - -# ToolConfig keys excluded -- per-call timing knob, analogous to inference_parameters.timeout. _EXCLUDED_TOOL_CONFIG_KEYS: frozenset[str] = frozenset({"timeout_sec"}) - -# HuggingFaceSeedSource keys excluded -- auth and endpoint URL are not data identity. _EXCLUDED_HF_SEED_KEYS: frozenset[str] = frozenset({"token", "endpoint"}) +# Optional collections whose `None` and `[]` representations must collapse so +# that builder-API and YAML-loaded configs producing identical datasets +# fingerprint identically. +_TOP_LEVEL_OPTIONAL_COLLECTIONS: frozenset[str] = frozenset( + {"model_configs", "tool_configs", "constraints", "processors"} +) +_TOOL_CONFIG_OPTIONAL_COLLECTIONS: frozenset[str] = frozenset({"allow_tools"}) + # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- -def fingerprint_config( - config: DataDesignerConfig, - *, - custom_column_source: bool = False, -) -> dict[str, str | int]: +def fingerprint_config(config: DataDesignerConfig) -> dict[str, str | int]: """Compute a deterministic fingerprint of a workflow config. The fingerprint is content-addressable: identical configs (modulo excluded @@ -72,56 +79,37 @@ def fingerprint_config( changing an excluded field does not. Identity-relevant fields: - * ``columns`` (names, types, generator params, processors, validators, - skip/drop/propagate_skip flags) - * ``model_configs`` (alias, model, provider, sampling-relevant inference - params -- temperature, top_p, max_tokens, extra_body) - * ``tool_configs`` (alias, providers, allow_tools, max_tool_call_turns): - the set of MCP tools an LLM can call shapes what it produces - * ``seed_config`` (source path / sampling strategy / selection strategy) - * ``constraints`` - * top-level ``processors`` - - Excluded fields: - * ``profilers`` (post-generation analysis) - * ``model_configs[*].skip_health_check`` - * ``inference_parameters.max_parallel_requests``, ``inference_parameters.timeout`` - * ``tool_configs[*].timeout_sec`` (per-call timing knob, not output identity) - * HuggingFace seed source ``token`` and ``endpoint`` - - Custom column generators are always identified by registered function name - and ``generator_params`` (L1). When ``custom_column_source=True``, the - function source is also hashed (L2); plugins whose source cannot be - retrieved degrade gracefully with a warning. - - Note: ``buffer_size`` lives on :class:`RunConfig`, not on - :class:`DataDesignerConfig`, and is therefore not part of this fingerprint. - The fingerprint identifies the workflow definition; runtime knobs that - don't change the final dataset are out of scope. - - Limitations: - * **L1 collisions on ``__name__``**: custom columns are identified at L1 - by the generator's bare ``__name__``, not its qualified module path. - Two unrelated generators in different modules with the same name and - identical ``generator_params`` will produce the same L1 hash. Pass - ``custom_column_source=True`` to disambiguate via source. - * **L2 hashes raw source**: comment-only and formatting changes to a - generator's source will change the L2 hash even though they don't - affect behavior. + * `columns` - names, types, generator params, processors, validators, + skip/drop flags. Column order is part of identity (DAG ordering). + * `model_configs` - alias, model, provider, sampling-relevant inference + params (temperature, top_p, max_tokens, extra_body). Sorted by alias. + * `tool_configs` - alias, providers, allow_tools, max_tool_call_turns + (the set of MCP tools shapes generation). Sorted by tool_alias. + * `seed_config` - source path, sampling strategy, selection strategy. + * `constraints`, top-level `processors`. + + See module-level constants for the canonical excluded-fields table. + + Custom column generators contribute their function's `__name__`, + `__qualname__`, `__module__`, `generator_params`, and the decorator + metadata set by `@custom_column_generator()` (`required_columns`, + `side_effect_columns`, `model_aliases`). + + Limitation: closures captured via factory functions (e.g. `make_gen(factor)` + returning a `gen` whose body references `factor`) share `__name__`, + `__qualname__`, `__module__`, and source text, so two closures with + different captured state will fingerprint identically. The fingerprint + cannot see closure cell values. Args: config: The workflow config to fingerprint. - custom_column_source: If True, also hash ``inspect.getsource()`` of - each custom column generator (L2). Defaults to False (L1 only). Returns: - A dict with ``config_hash`` (``"sha256:..."``), ``config_hash_algo``, - and ``config_hash_version`` suitable for embedding in dataset metadata. + A dict with `config_hash` (`"sha256:..."`), `config_hash_algo`, and + `config_hash_version` suitable for embedding in dataset metadata. """ - payload: dict[str, Any] = {"config": _normalize_config_dict(config.to_dict())} - if custom_column_source: - payload["custom_column_sources"] = _collect_custom_column_sources(config) - # No `default=` fallback: a non-JSON-native value would silently break determinism (e.g., repr with memory addresses). + payload = _normalize_config_dict(config.to_dict(), config) + # No `default=` fallback: a non-JSON-native value would silently break determinism (e.g. repr with memory addresses). canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() return { @@ -136,8 +124,19 @@ def fingerprint_config( # --------------------------------------------------------------------------- -def _drop_keys(source: dict[str, Any], keys: frozenset[str]) -> dict[str, Any]: - return {k: v for k, v in source.items() if k not in keys} +def _drop_keys(source: dict[str, Any], keys: Iterable[str]) -> dict[str, Any]: + keyset = set(keys) + return {k: v for k, v in source.items() if k not in keyset} + + +def _drop_empty_optional(source: dict[str, Any], keys: Iterable[str]) -> dict[str, Any]: + """Drop keys whose value is `None` or an empty list. + + `None` and `[]` are user-equivalent for optional collection fields; this + collapses both to "absent" before hashing. + """ + keyset = set(keys) + return {k: v for k, v in source.items() if not (k in keyset and (v is None or v == []))} def _normalize_model_config(model_config: dict[str, Any]) -> dict[str, Any]: @@ -148,6 +147,11 @@ def _normalize_model_config(model_config: dict[str, Any]) -> dict[str, Any]: return normalized +def _normalize_tool_config(tool_config: dict[str, Any]) -> dict[str, Any]: + normalized = _drop_keys(tool_config, _EXCLUDED_TOOL_CONFIG_KEYS) + return _drop_empty_optional(normalized, _TOOL_CONFIG_OPTIONAL_COLLECTIONS) + + def _normalize_seed_config(seed_config: dict[str, Any]) -> dict[str, Any]: normalized = dict(seed_config) seed_source = normalized.get("source") @@ -156,54 +160,56 @@ def _normalize_seed_config(seed_config: dict[str, Any]) -> dict[str, Any]: return normalized -def _normalize_tool_config(tool_config: dict[str, Any]) -> dict[str, Any]: - return _drop_keys(tool_config, _EXCLUDED_TOOL_CONFIG_KEYS) - +def _enrich_custom_columns(config: DataDesignerConfig, columns_dump: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Replace each custom column's serialized `generator_function` (just the + bare `__name__`) with a richer identity dict that includes `__qualname__`, + `__module__`, and the `@custom_column_generator()` decorator metadata. -def _normalize_config_dict(config_dict: dict[str, Any]) -> dict[str, Any]: + Walks `config.columns` and `columns_dump` in lockstep so positional + correspondence is reliable. + """ + enriched: list[dict[str, Any]] = [] + for col, dumped in zip(config.columns, columns_dump): + if isinstance(col, CustomColumnConfig): + fn = col.generator_function + metadata = getattr(fn, "custom_column_metadata", {}) or {} + dumped = { + **dumped, + "generator_function": { + "name": getattr(fn, "__name__", None), + "qualname": getattr(fn, "__qualname__", None), + "module": getattr(fn, "__module__", None), + "metadata": metadata, + }, + } + enriched.append(dumped) + return enriched + + +def _normalize_config_dict(config_dict: dict[str, Any], config: DataDesignerConfig) -> dict[str, Any]: normalized = _drop_keys(config_dict, _EXCLUDED_TOP_LEVEL_KEYS) + normalized = _drop_empty_optional(normalized, _TOP_LEVEL_OPTIONAL_COLLECTIONS) + + columns = normalized.get("columns") + if columns: + normalized["columns"] = _enrich_custom_columns(config, columns) + model_configs = normalized.get("model_configs") if model_configs: - normalized["model_configs"] = [_normalize_model_config(mc) for mc in model_configs] + normalized["model_configs"] = sorted( + (_normalize_model_config(mc) for mc in model_configs), + key=lambda mc: mc.get("alias", ""), + ) + tool_configs = normalized.get("tool_configs") if tool_configs: - normalized["tool_configs"] = [_normalize_tool_config(tc) for tc in tool_configs] + normalized["tool_configs"] = sorted( + (_normalize_tool_config(tc) for tc in tool_configs), + key=lambda tc: tc.get("tool_alias", ""), + ) + seed_config = normalized.get("seed_config") if seed_config: normalized["seed_config"] = _normalize_seed_config(seed_config) - return normalized - - -def _hash_custom_column_source(fn: Callable[..., Any], column_name: str) -> str | None: - """Hash the source of a custom column generator (L2). - Returns the sha256 hex digest of the function source, or ``None`` if the - source cannot be retrieved (e.g., compiled / zipped plugin, C extension, - interactively-defined function). Plugins that can't be source-hashed - degrade gracefully with a warning rather than raising. - """ - try: - unwrapped = inspect.unwrap(fn) - source = inspect.getsource(unwrapped) - except (OSError, TypeError) as exc: - logger.warning( - "Could not retrieve source for custom column %r generator (%s); " - "fingerprint will not detect implementation changes for this column.", - column_name, - exc, - ) - return None - return hashlib.sha256(source.encode("utf-8")).hexdigest() - - -def _collect_custom_column_sources(config: DataDesignerConfig) -> list[dict[str, Any]]: - sources: list[dict[str, Any]] = [] - for col in config.columns: - if isinstance(col, CustomColumnConfig): - sources.append( - { - "name": col.name, - "source_hash": _hash_custom_column_source(col.generator_function, col.name), - } - ) - return sources + return normalized diff --git a/packages/data-designer-config/tests/config/test_fingerprint.py b/packages/data-designer-config/tests/config/test_fingerprint.py index 0e5f4dc48..b05a6f540 100644 --- a/packages/data-designer-config/tests/config/test_fingerprint.py +++ b/packages/data-designer-config/tests/config/test_fingerprint.py @@ -3,17 +3,14 @@ from __future__ import annotations -import inspect import subprocess import sys -from collections.abc import Callable from typing import Any import pytest import yaml from pydantic import BaseModel -from data_designer.config import fingerprint as fp_mod from data_designer.config.analysis.column_profilers import JudgeScoreProfilerConfig from data_designer.config.base import SkipConfig from data_designer.config.column_configs import ( @@ -37,8 +34,8 @@ from data_designer.config.seed_source import HuggingFaceSeedSource -def _hash(config: DataDesignerConfig, *, custom_column_source: bool = False) -> str: - return str(fingerprint_config(config, custom_column_source=custom_column_source)["config_hash"]) +def _hash(config: DataDesignerConfig) -> str: + return str(fingerprint_config(config)["config_hash"]) def test_fingerprint_shape(stub_data_designer_config: DataDesignerConfig) -> None: @@ -82,10 +79,10 @@ def test_fingerprint_deterministic_across_processes(stub_data_designer_config_st # --------------------------------------------------------------------------- -def _make_model() -> ModelConfig: +def _make_model(alias: str = "m", model: str = "some-model") -> ModelConfig: return ModelConfig( - alias="m", - model="some-model", + alias=alias, + model=model, inference_parameters=ChatCompletionInferenceParams(temperature=0.5, top_p=0.9, max_tokens=128), ) @@ -151,6 +148,7 @@ def test_changing_temperature_changes_hash() -> None: def test_changing_column_order_changes_hash() -> None: + """Column order is part of identity (DAG ordering).""" cols_a = [ SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), SamplerColumnConfig(name="y", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), @@ -379,7 +377,58 @@ def test_changing_hf_seed_path_changes_hash() -> None: # --------------------------------------------------------------------------- -# Custom columns: L1 (default) and L2 (opt-in source hashing). +# Canonicalization: alias-keyed lookup tables are order-independent, and +# `None`/empty-list optional collections collapse to a single representation. +# --------------------------------------------------------------------------- + + +def test_model_configs_order_independent() -> None: + """`model_configs` is alias-keyed; reordering the list must not flip the hash.""" + a = _make_minimal_config(model_configs=[_make_model("m1"), _make_model("m2")]) + b = _make_minimal_config(model_configs=[_make_model("m2"), _make_model("m1")]) + assert _hash(a) == _hash(b) + + +def test_tool_configs_order_independent() -> None: + """`tool_configs` is alias-keyed; reordering the list must not flip the hash.""" + a = _make_minimal_config( + tool_configs=[ + ToolConfig(tool_alias="t1", providers=["p"]), + ToolConfig(tool_alias="t2", providers=["p"]), + ], + ) + b = _make_minimal_config( + tool_configs=[ + ToolConfig(tool_alias="t2", providers=["p"]), + ToolConfig(tool_alias="t1", providers=["p"]), + ], + ) + assert _hash(a) == _hash(b) + + +@pytest.mark.parametrize( + "field", + ["model_configs", "tool_configs", "constraints", "processors"], +) +def test_none_vs_empty_list_for_optional_top_level_fields_match(field: str) -> None: + """`None` and `[]` must produce identical hashes for optional top-level collections.""" + base_kwargs: dict[str, Any] = { + "columns": [SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1))], + } + a = DataDesignerConfig(**base_kwargs, **{field: None}) + b = DataDesignerConfig(**base_kwargs, **{field: []}) + assert _hash(a) == _hash(b) + + +def test_tool_config_allow_tools_none_vs_empty_match() -> None: + """`allow_tools=None` and `allow_tools=[]` must produce identical hashes.""" + a = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"], allow_tools=None)]) + b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"], allow_tools=[])]) + assert _hash(a) == _hash(b) + + +# --------------------------------------------------------------------------- +# Custom column identity: name + qualname + module + decorator metadata. # --------------------------------------------------------------------------- @@ -397,7 +446,7 @@ def _generate_v2(row: dict, generator_params: _GenParamsV1) -> str: # pragma: n return str(row.get("x", 0) * generator_params.factor + 1) -def _make_custom_config(fn: Callable[..., Any], params: _GenParamsV1 | None = None) -> DataDesignerConfig: +def _make_custom_config(fn: Any, params: _GenParamsV1 | None = None) -> DataDesignerConfig: return _make_minimal_config( columns=[ SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), @@ -410,50 +459,62 @@ def _make_custom_config(fn: Callable[..., Any], params: _GenParamsV1 | None = No ) -def test_custom_column_l1_includes_generator_params() -> None: +def test_custom_column_includes_generator_params() -> None: a = _make_custom_config(_generate_v1, _GenParamsV1(factor=1)) b = _make_custom_config(_generate_v1, _GenParamsV1(factor=2)) assert _hash(a) != _hash(b) -def test_custom_column_l1_includes_generator_function_name() -> None: +def test_custom_column_includes_generator_function_name() -> None: a = _make_custom_config(_generate_v1) b = _make_custom_config(_generate_v2) - # Different function names serialize to different values via field_serializer. assert _hash(a) != _hash(b) -def test_custom_column_l2_detects_source_change(monkeypatch: pytest.MonkeyPatch) -> None: - a = _make_custom_config(_generate_v1) - base = _hash(a, custom_column_source=True) +def test_custom_column_qualname_disambiguates_same_name() -> None: + """Two functions sharing `__name__` but with different `__qualname__` must + produce different hashes (the fix for the same-name-different-scope + collision class).""" - # Simulate an implementation edit by feeding a different source string. - sources = iter(["original-source", "edited-source"]) - monkeypatch.setattr(fp_mod, "_hash_custom_column_source", lambda fn, name: next(sources)) + def _make_outer_a() -> Any: + @custom_column_generator() + def _gen(row: dict, generator_params: _GenParamsV1) -> str: # pragma: no cover + return "" - edit_first = _hash(a, custom_column_source=True) - edit_second = _hash(a, custom_column_source=True) - assert edit_first != edit_second - assert base != edit_first + return _gen + def _make_outer_b() -> Any: + @custom_column_generator() + def _gen(row: dict, generator_params: _GenParamsV1) -> str: # pragma: no cover + return "" -def test_custom_column_unhashable_source_degrades_gracefully( - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture -) -> None: - """A plugin whose source can't be retrieved should warn, not raise.""" + return _gen - def _raise_oserror(_fn: object) -> str: - raise OSError("compiled / zipped plugin") + fn_a = _make_outer_a() + fn_b = _make_outer_b() + assert fn_a.__name__ == fn_b.__name__ + assert fn_a.__qualname__ != fn_b.__qualname__ - monkeypatch.setattr(inspect, "getsource", _raise_oserror) + assert _hash(_make_custom_config(fn_a)) != _hash(_make_custom_config(fn_b)) - a = _make_custom_config(_generate_v1) - with caplog.at_level("WARNING", logger=fp_mod.__name__): - out = a.fingerprint(custom_column_source=True) - assert out["config_hash"].startswith(f"{CONFIG_HASH_ALGO}:") - assert "Could not retrieve source" in caplog.text +def test_custom_column_decorator_metadata_changes_hash() -> None: + """Two generators sharing `__name__` and `__qualname__` but with different + `@custom_column_generator()` metadata (`required_columns` etc.) must + produce different hashes — `required_columns` changes DAG order and + `side_effect_columns` changes the output schema.""" -def test_l1_and_l2_produce_different_hashes() -> None: - a = _make_custom_config(_generate_v1) - assert _hash(a) != _hash(a, custom_column_source=True) + def _make_with_required(required_cols: list[str]) -> Any: + @custom_column_generator(required_columns=required_cols) + def _gen(row: dict, generator_params: _GenParamsV1) -> str: # pragma: no cover + return "" + + return _gen + + fn_a = _make_with_required(["x"]) + fn_b = _make_with_required(["x", "y"]) + assert fn_a.__name__ == fn_b.__name__ + assert fn_a.__qualname__ == fn_b.__qualname__ + assert fn_a.custom_column_metadata != fn_b.custom_column_metadata + + assert _hash(_make_custom_config(fn_a)) != _hash(_make_custom_config(fn_b)) From 2debe120dca911be0c8e310aeb440cfec3bd48ca Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Thu, 30 Apr 2026 13:11:54 -0600 Subject: [PATCH 6/7] test(fingerprint): rename _hash helper to _compute_hash Function names should be action words; `_hash` is a noun. Rename the test-only helper to `_compute_hash` to match its verb-form behavior (it computes a hash from a config). No behavioral change. Signed-off-by: Nabin Mulepati Made-with: Cursor --- .../tests/config/test_fingerprint.py | 126 +++++++++--------- 1 file changed, 64 insertions(+), 62 deletions(-) diff --git a/packages/data-designer-config/tests/config/test_fingerprint.py b/packages/data-designer-config/tests/config/test_fingerprint.py index b05a6f540..c8ca4a81a 100644 --- a/packages/data-designer-config/tests/config/test_fingerprint.py +++ b/packages/data-designer-config/tests/config/test_fingerprint.py @@ -34,7 +34,7 @@ from data_designer.config.seed_source import HuggingFaceSeedSource -def _hash(config: DataDesignerConfig) -> str: +def _compute_hash(config: DataDesignerConfig) -> str: return str(fingerprint_config(config)["config_hash"]) @@ -54,7 +54,7 @@ def test_fingerprint_deterministic_within_process( stub_data_designer_config_str: str, ) -> None: rebuilt = DataDesignerConfig.model_validate(yaml.safe_load(stub_data_designer_config_str)) - assert _hash(stub_data_designer_config) == _hash(rebuilt) + assert _compute_hash(stub_data_designer_config) == _compute_hash(rebuilt) def test_fingerprint_deterministic_across_processes(stub_data_designer_config_str: str) -> None: @@ -71,7 +71,7 @@ def test_fingerprint_deterministic_across_processes(stub_data_designer_config_st out = result.stdout.strip() cfg = DataDesignerConfig.model_validate(yaml.safe_load(stub_data_designer_config_str)) - assert out == _hash(cfg) + assert out == _compute_hash(cfg) # --------------------------------------------------------------------------- @@ -101,39 +101,39 @@ def _make_minimal_config(**overrides: object) -> DataDesignerConfig: # --------------------------------------------------------------------------- -def test_changing_column_name_changes_hash() -> None: +def test_changing_column_name_changes_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( columns=[SamplerColumnConfig(name="y", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1))], ) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_column_type_changes_hash() -> None: +def test_changing_column_type_changes_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( columns=[ SamplerColumnConfig(name="x", sampler_type="category", params=CategorySamplerParams(values=["a", "b"])), ], ) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_sampler_params_changes_hash() -> None: +def test_changing_sampler_params_changes_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( columns=[SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=2))], ) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_model_identity_changes_hash() -> None: +def test_changing_model_identity_changes_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config(model_configs=[ModelConfig(alias="m", model="other-model")]) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_temperature_changes_hash() -> None: +def test_changing_temperature_changes_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -144,20 +144,20 @@ def test_changing_temperature_changes_hash() -> None: ) ], ) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_column_order_changes_hash() -> None: +def test_changing_column_order_changes_compute_hash() -> None: """Column order is part of identity (DAG ordering).""" cols_a = [ SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), SamplerColumnConfig(name="y", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), ] cols_b = list(reversed(cols_a)) - assert _hash(_make_minimal_config(columns=cols_a)) != _hash(_make_minimal_config(columns=cols_b)) + assert _compute_hash(_make_minimal_config(columns=cols_a)) != _compute_hash(_make_minimal_config(columns=cols_b)) -def test_changing_skip_changes_hash() -> None: +def test_changing_skip_changes_compute_hash() -> None: base_col = LLMTextColumnConfig(name="t", prompt="hi {{x}}", model_alias="m") skipped = LLMTextColumnConfig( name="t", @@ -173,24 +173,26 @@ def test_changing_skip_changes_hash() -> None: SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), skipped, ] - assert _hash(_make_minimal_config(columns=cols_no_skip)) != _hash(_make_minimal_config(columns=cols_skip)) + assert _compute_hash(_make_minimal_config(columns=cols_no_skip)) != _compute_hash( + _make_minimal_config(columns=cols_skip) + ) -def test_changing_constraint_changes_hash() -> None: +def test_changing_constraint_changes_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( constraints=[ScalarInequalityConstraint(target_column="x", operator=InequalityOperator.LT, rhs=0.5)], ) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_top_level_processor_changes_hash() -> None: +def test_changing_top_level_processor_changes_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config(processors=[DropColumnsProcessorConfig(name="drop", column_names=["x"])]) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_extra_body_changes_hash() -> None: +def test_changing_extra_body_changes_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -203,10 +205,10 @@ def test_changing_extra_body_changes_hash() -> None: ) ], ) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_provider_changes_hash() -> None: +def test_changing_provider_changes_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -218,10 +220,10 @@ def test_changing_provider_changes_hash() -> None: ) ], ) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_sampling_strategy_changes_hash() -> None: +def test_changing_sampling_strategy_changes_compute_hash() -> None: a = _make_minimal_config( seed_config=SeedConfig( source=HuggingFaceSeedSource(path="datasets/x/y/data.csv"), @@ -234,10 +236,10 @@ def test_changing_sampling_strategy_changes_hash() -> None: sampling_strategy=SamplingStrategy.SHUFFLE, ), ) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_selection_strategy_changes_hash() -> None: +def test_changing_selection_strategy_changes_compute_hash() -> None: a = _make_minimal_config( seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/data.csv")), ) @@ -247,45 +249,45 @@ def test_changing_selection_strategy_changes_hash() -> None: selection_strategy=IndexRange(start=0, end=99), ), ) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_adding_tool_config_changes_hash() -> None: +def test_adding_tool_config_changes_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"])]) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_tool_config_alias_changes_hash() -> None: +def test_changing_tool_config_alias_changes_compute_hash() -> None: a = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t1", providers=["p"])]) b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t2", providers=["p"])]) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_tool_config_providers_changes_hash() -> None: +def test_changing_tool_config_providers_changes_compute_hash() -> None: a = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p1"])]) b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p2"])]) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_tool_config_allow_tools_changes_hash() -> None: +def test_changing_tool_config_allow_tools_changes_compute_hash() -> None: a = _make_minimal_config( tool_configs=[ToolConfig(tool_alias="t", providers=["p"], allow_tools=["search"])], ) b = _make_minimal_config( tool_configs=[ToolConfig(tool_alias="t", providers=["p"], allow_tools=["search", "list"])], ) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) -def test_changing_max_tool_call_turns_changes_hash() -> None: +def test_changing_max_tool_call_turns_changes_compute_hash() -> None: a = _make_minimal_config( tool_configs=[ToolConfig(tool_alias="t", providers=["p"], max_tool_call_turns=5)], ) b = _make_minimal_config( tool_configs=[ToolConfig(tool_alias="t", providers=["p"], max_tool_call_turns=10)], ) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) # --------------------------------------------------------------------------- @@ -293,7 +295,7 @@ def test_changing_max_tool_call_turns_changes_hash() -> None: # --------------------------------------------------------------------------- -def test_skip_health_check_does_not_change_hash() -> None: +def test_skip_health_check_does_not_change_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -305,10 +307,10 @@ def test_skip_health_check_does_not_change_hash() -> None: ) ], ) - assert _hash(a) == _hash(b) + assert _compute_hash(a) == _compute_hash(b) -def test_max_parallel_requests_does_not_change_hash() -> None: +def test_max_parallel_requests_does_not_change_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -321,10 +323,10 @@ def test_max_parallel_requests_does_not_change_hash() -> None: ) ], ) - assert _hash(a) == _hash(b) + assert _compute_hash(a) == _compute_hash(b) -def test_inference_timeout_does_not_change_hash() -> None: +def test_inference_timeout_does_not_change_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -337,24 +339,24 @@ def test_inference_timeout_does_not_change_hash() -> None: ) ], ) - assert _hash(a) == _hash(b) + assert _compute_hash(a) == _compute_hash(b) -def test_tool_config_timeout_sec_does_not_change_hash() -> None: +def test_tool_config_timeout_sec_does_not_change_compute_hash() -> None: a = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"])]) b = _make_minimal_config( tool_configs=[ToolConfig(tool_alias="t", providers=["p"], timeout_sec=30.0)], ) - assert _hash(a) == _hash(b) + assert _compute_hash(a) == _compute_hash(b) -def test_profilers_do_not_change_hash() -> None: +def test_profilers_do_not_change_compute_hash() -> None: a = _make_minimal_config() b = _make_minimal_config(profilers=[JudgeScoreProfilerConfig(model_alias="m")]) - assert _hash(a) == _hash(b) + assert _compute_hash(a) == _compute_hash(b) -def test_hf_seed_token_and_endpoint_do_not_change_hash() -> None: +def test_hf_seed_token_and_endpoint_do_not_change_compute_hash() -> None: a = _make_minimal_config( seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/data.csv")), ) @@ -367,13 +369,13 @@ def test_hf_seed_token_and_endpoint_do_not_change_hash() -> None: ), ), ) - assert _hash(a) == _hash(b) + assert _compute_hash(a) == _compute_hash(b) -def test_changing_hf_seed_path_changes_hash() -> None: +def test_changing_hf_seed_path_changes_compute_hash() -> None: a = _make_minimal_config(seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/a.csv"))) b = _make_minimal_config(seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/b.csv"))) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) # --------------------------------------------------------------------------- @@ -386,7 +388,7 @@ def test_model_configs_order_independent() -> None: """`model_configs` is alias-keyed; reordering the list must not flip the hash.""" a = _make_minimal_config(model_configs=[_make_model("m1"), _make_model("m2")]) b = _make_minimal_config(model_configs=[_make_model("m2"), _make_model("m1")]) - assert _hash(a) == _hash(b) + assert _compute_hash(a) == _compute_hash(b) def test_tool_configs_order_independent() -> None: @@ -403,7 +405,7 @@ def test_tool_configs_order_independent() -> None: ToolConfig(tool_alias="t1", providers=["p"]), ], ) - assert _hash(a) == _hash(b) + assert _compute_hash(a) == _compute_hash(b) @pytest.mark.parametrize( @@ -417,14 +419,14 @@ def test_none_vs_empty_list_for_optional_top_level_fields_match(field: str) -> N } a = DataDesignerConfig(**base_kwargs, **{field: None}) b = DataDesignerConfig(**base_kwargs, **{field: []}) - assert _hash(a) == _hash(b) + assert _compute_hash(a) == _compute_hash(b) def test_tool_config_allow_tools_none_vs_empty_match() -> None: """`allow_tools=None` and `allow_tools=[]` must produce identical hashes.""" a = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"], allow_tools=None)]) b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"], allow_tools=[])]) - assert _hash(a) == _hash(b) + assert _compute_hash(a) == _compute_hash(b) # --------------------------------------------------------------------------- @@ -462,13 +464,13 @@ def _make_custom_config(fn: Any, params: _GenParamsV1 | None = None) -> DataDesi def test_custom_column_includes_generator_params() -> None: a = _make_custom_config(_generate_v1, _GenParamsV1(factor=1)) b = _make_custom_config(_generate_v1, _GenParamsV1(factor=2)) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) def test_custom_column_includes_generator_function_name() -> None: a = _make_custom_config(_generate_v1) b = _make_custom_config(_generate_v2) - assert _hash(a) != _hash(b) + assert _compute_hash(a) != _compute_hash(b) def test_custom_column_qualname_disambiguates_same_name() -> None: @@ -495,10 +497,10 @@ def _gen(row: dict, generator_params: _GenParamsV1) -> str: # pragma: no cover assert fn_a.__name__ == fn_b.__name__ assert fn_a.__qualname__ != fn_b.__qualname__ - assert _hash(_make_custom_config(fn_a)) != _hash(_make_custom_config(fn_b)) + assert _compute_hash(_make_custom_config(fn_a)) != _compute_hash(_make_custom_config(fn_b)) -def test_custom_column_decorator_metadata_changes_hash() -> None: +def test_custom_column_decorator_metadata_changes_compute_hash() -> None: """Two generators sharing `__name__` and `__qualname__` but with different `@custom_column_generator()` metadata (`required_columns` etc.) must produce different hashes — `required_columns` changes DAG order and @@ -517,4 +519,4 @@ def _gen(row: dict, generator_params: _GenParamsV1) -> str: # pragma: no cover assert fn_a.__qualname__ == fn_b.__qualname__ assert fn_a.custom_column_metadata != fn_b.custom_column_metadata - assert _hash(_make_custom_config(fn_a)) != _hash(_make_custom_config(fn_b)) + assert _compute_hash(_make_custom_config(fn_a)) != _compute_hash(_make_custom_config(fn_b)) From 180ecb0366f27b241702ee1348f8f10e73cb0864 Mon Sep 17 00:00:00 2001 From: Nabin Mulepati Date: Thu, 30 Apr 2026 13:53:26 -0600 Subject: [PATCH 7/7] test(fingerprint): pin closure-capture limitation; restore test names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous _hash -> _compute_hash blanket rename also caught the test names that happen to end in "_hash()" (e.g. test_changing_X_changes_hash). "hash" is a noun there — it describes what the test is about, not the helper being called. Restore the original names; only the helper itself stays renamed. Add `test_closure_captured_state_is_a_known_limitation` per @johnnygreco's approval follow-up: factory-built closures with different captured state share __name__/__qualname__/__module__/source and so fingerprint identically. Pin that behavior so a future change either keeps the limitation or has to delete the matching docstring paragraph in lockstep. Signed-off-by: Nabin Mulepati Made-with: Cursor --- .../tests/config/test_fingerprint.py | 78 ++++++++++++------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/packages/data-designer-config/tests/config/test_fingerprint.py b/packages/data-designer-config/tests/config/test_fingerprint.py index c8ca4a81a..0535de704 100644 --- a/packages/data-designer-config/tests/config/test_fingerprint.py +++ b/packages/data-designer-config/tests/config/test_fingerprint.py @@ -101,7 +101,7 @@ def _make_minimal_config(**overrides: object) -> DataDesignerConfig: # --------------------------------------------------------------------------- -def test_changing_column_name_changes_compute_hash() -> None: +def test_changing_column_name_changes_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( columns=[SamplerColumnConfig(name="y", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1))], @@ -109,7 +109,7 @@ def test_changing_column_name_changes_compute_hash() -> None: assert _compute_hash(a) != _compute_hash(b) -def test_changing_column_type_changes_compute_hash() -> None: +def test_changing_column_type_changes_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( columns=[ @@ -119,7 +119,7 @@ def test_changing_column_type_changes_compute_hash() -> None: assert _compute_hash(a) != _compute_hash(b) -def test_changing_sampler_params_changes_compute_hash() -> None: +def test_changing_sampler_params_changes_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( columns=[SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=2))], @@ -127,13 +127,13 @@ def test_changing_sampler_params_changes_compute_hash() -> None: assert _compute_hash(a) != _compute_hash(b) -def test_changing_model_identity_changes_compute_hash() -> None: +def test_changing_model_identity_changes_hash() -> None: a = _make_minimal_config() b = _make_minimal_config(model_configs=[ModelConfig(alias="m", model="other-model")]) assert _compute_hash(a) != _compute_hash(b) -def test_changing_temperature_changes_compute_hash() -> None: +def test_changing_temperature_changes_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -147,7 +147,7 @@ def test_changing_temperature_changes_compute_hash() -> None: assert _compute_hash(a) != _compute_hash(b) -def test_changing_column_order_changes_compute_hash() -> None: +def test_changing_column_order_changes_hash() -> None: """Column order is part of identity (DAG ordering).""" cols_a = [ SamplerColumnConfig(name="x", sampler_type="uniform", params=UniformSamplerParams(low=0, high=1)), @@ -157,7 +157,7 @@ def test_changing_column_order_changes_compute_hash() -> None: assert _compute_hash(_make_minimal_config(columns=cols_a)) != _compute_hash(_make_minimal_config(columns=cols_b)) -def test_changing_skip_changes_compute_hash() -> None: +def test_changing_skip_changes_hash() -> None: base_col = LLMTextColumnConfig(name="t", prompt="hi {{x}}", model_alias="m") skipped = LLMTextColumnConfig( name="t", @@ -178,7 +178,7 @@ def test_changing_skip_changes_compute_hash() -> None: ) -def test_changing_constraint_changes_compute_hash() -> None: +def test_changing_constraint_changes_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( constraints=[ScalarInequalityConstraint(target_column="x", operator=InequalityOperator.LT, rhs=0.5)], @@ -186,13 +186,13 @@ def test_changing_constraint_changes_compute_hash() -> None: assert _compute_hash(a) != _compute_hash(b) -def test_changing_top_level_processor_changes_compute_hash() -> None: +def test_changing_top_level_processor_changes_hash() -> None: a = _make_minimal_config() b = _make_minimal_config(processors=[DropColumnsProcessorConfig(name="drop", column_names=["x"])]) assert _compute_hash(a) != _compute_hash(b) -def test_changing_extra_body_changes_compute_hash() -> None: +def test_changing_extra_body_changes_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -208,7 +208,7 @@ def test_changing_extra_body_changes_compute_hash() -> None: assert _compute_hash(a) != _compute_hash(b) -def test_changing_provider_changes_compute_hash() -> None: +def test_changing_provider_changes_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -223,7 +223,7 @@ def test_changing_provider_changes_compute_hash() -> None: assert _compute_hash(a) != _compute_hash(b) -def test_changing_sampling_strategy_changes_compute_hash() -> None: +def test_changing_sampling_strategy_changes_hash() -> None: a = _make_minimal_config( seed_config=SeedConfig( source=HuggingFaceSeedSource(path="datasets/x/y/data.csv"), @@ -239,7 +239,7 @@ def test_changing_sampling_strategy_changes_compute_hash() -> None: assert _compute_hash(a) != _compute_hash(b) -def test_changing_selection_strategy_changes_compute_hash() -> None: +def test_changing_selection_strategy_changes_hash() -> None: a = _make_minimal_config( seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/data.csv")), ) @@ -252,25 +252,25 @@ def test_changing_selection_strategy_changes_compute_hash() -> None: assert _compute_hash(a) != _compute_hash(b) -def test_adding_tool_config_changes_compute_hash() -> None: +def test_adding_tool_config_changes_hash() -> None: a = _make_minimal_config() b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"])]) assert _compute_hash(a) != _compute_hash(b) -def test_changing_tool_config_alias_changes_compute_hash() -> None: +def test_changing_tool_config_alias_changes_hash() -> None: a = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t1", providers=["p"])]) b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t2", providers=["p"])]) assert _compute_hash(a) != _compute_hash(b) -def test_changing_tool_config_providers_changes_compute_hash() -> None: +def test_changing_tool_config_providers_changes_hash() -> None: a = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p1"])]) b = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p2"])]) assert _compute_hash(a) != _compute_hash(b) -def test_changing_tool_config_allow_tools_changes_compute_hash() -> None: +def test_changing_tool_config_allow_tools_changes_hash() -> None: a = _make_minimal_config( tool_configs=[ToolConfig(tool_alias="t", providers=["p"], allow_tools=["search"])], ) @@ -280,7 +280,7 @@ def test_changing_tool_config_allow_tools_changes_compute_hash() -> None: assert _compute_hash(a) != _compute_hash(b) -def test_changing_max_tool_call_turns_changes_compute_hash() -> None: +def test_changing_max_tool_call_turns_changes_hash() -> None: a = _make_minimal_config( tool_configs=[ToolConfig(tool_alias="t", providers=["p"], max_tool_call_turns=5)], ) @@ -295,7 +295,7 @@ def test_changing_max_tool_call_turns_changes_compute_hash() -> None: # --------------------------------------------------------------------------- -def test_skip_health_check_does_not_change_compute_hash() -> None: +def test_skip_health_check_does_not_change_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -310,7 +310,7 @@ def test_skip_health_check_does_not_change_compute_hash() -> None: assert _compute_hash(a) == _compute_hash(b) -def test_max_parallel_requests_does_not_change_compute_hash() -> None: +def test_max_parallel_requests_does_not_change_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -326,7 +326,7 @@ def test_max_parallel_requests_does_not_change_compute_hash() -> None: assert _compute_hash(a) == _compute_hash(b) -def test_inference_timeout_does_not_change_compute_hash() -> None: +def test_inference_timeout_does_not_change_hash() -> None: a = _make_minimal_config() b = _make_minimal_config( model_configs=[ @@ -342,7 +342,7 @@ def test_inference_timeout_does_not_change_compute_hash() -> None: assert _compute_hash(a) == _compute_hash(b) -def test_tool_config_timeout_sec_does_not_change_compute_hash() -> None: +def test_tool_config_timeout_sec_does_not_change_hash() -> None: a = _make_minimal_config(tool_configs=[ToolConfig(tool_alias="t", providers=["p"])]) b = _make_minimal_config( tool_configs=[ToolConfig(tool_alias="t", providers=["p"], timeout_sec=30.0)], @@ -350,13 +350,13 @@ def test_tool_config_timeout_sec_does_not_change_compute_hash() -> None: assert _compute_hash(a) == _compute_hash(b) -def test_profilers_do_not_change_compute_hash() -> None: +def test_profilers_do_not_change_hash() -> None: a = _make_minimal_config() b = _make_minimal_config(profilers=[JudgeScoreProfilerConfig(model_alias="m")]) assert _compute_hash(a) == _compute_hash(b) -def test_hf_seed_token_and_endpoint_do_not_change_compute_hash() -> None: +def test_hf_seed_token_and_endpoint_do_not_change_hash() -> None: a = _make_minimal_config( seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/data.csv")), ) @@ -372,7 +372,7 @@ def test_hf_seed_token_and_endpoint_do_not_change_compute_hash() -> None: assert _compute_hash(a) == _compute_hash(b) -def test_changing_hf_seed_path_changes_compute_hash() -> None: +def test_changing_hf_seed_path_changes_hash() -> None: a = _make_minimal_config(seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/a.csv"))) b = _make_minimal_config(seed_config=SeedConfig(source=HuggingFaceSeedSource(path="datasets/x/y/b.csv"))) assert _compute_hash(a) != _compute_hash(b) @@ -500,7 +500,7 @@ def _gen(row: dict, generator_params: _GenParamsV1) -> str: # pragma: no cover assert _compute_hash(_make_custom_config(fn_a)) != _compute_hash(_make_custom_config(fn_b)) -def test_custom_column_decorator_metadata_changes_compute_hash() -> None: +def test_custom_column_decorator_metadata_changes_hash() -> None: """Two generators sharing `__name__` and `__qualname__` but with different `@custom_column_generator()` metadata (`required_columns` etc.) must produce different hashes — `required_columns` changes DAG order and @@ -520,3 +520,29 @@ def _gen(row: dict, generator_params: _GenParamsV1) -> str: # pragma: no cover assert fn_a.custom_column_metadata != fn_b.custom_column_metadata assert _compute_hash(_make_custom_config(fn_a)) != _compute_hash(_make_custom_config(fn_b)) + + +def test_closure_captured_state_is_a_known_limitation() -> None: + """Pin the documented closure-capture limitation. + + Factory-built closures with different captured state share `__name__`, + `__qualname__`, `__module__`, and source, so they fingerprint identically. + If this test ever flips, update or remove the closure-capture Limitation + block in `fingerprint_config()`'s docstring (and the matching note in the + PR description / public docs) so the contract and the implementation stay + in sync. + """ + + def _make_factor_gen(factor: int) -> Any: + @custom_column_generator() + def _gen(row: dict, generator_params: _GenParamsV1) -> str: # pragma: no cover + return str(row.get("x", 0) * factor) + + return _gen + + fn_a = _make_factor_gen(2) + fn_b = _make_factor_gen(7) + assert fn_a.__name__ == fn_b.__name__ + assert fn_a.__qualname__ == fn_b.__qualname__ + + assert _compute_hash(_make_custom_config(fn_a)) == _compute_hash(_make_custom_config(fn_b))