From fa85342bdd459fd52dd7ebc150b4382a0312875a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vojt=C4=9Bch=20C=C3=ADfka?= <550433@mail.muni.cz> Date: Fri, 19 Jun 2026 15:55:23 +0200 Subject: [PATCH 1/2] fix: isolate embedding staging dir to prevent cross-model contamination Embedding jobs wrote to a shared, persistent output_dir (embeddings/{split}/tiles) reused across models. Concurrent virchow2 and provgigapath runs raced in that dir, so log_artifacts uploaded both models' files into each run's mlflow artifact. Because both models embed the same tile grid (identical slide_id,x,y keys), this surfaced as mixed embedding dimensions (2560 + 1536) within a single artifact: a reshape crash for provgigapath, and inflated many-to-many joins for virchow2. Write each split into a per-run tempfile.TemporaryDirectory under output_dir instead. Runs no longer share a path, so concurrent jobs cannot contaminate each other, and the staging dir is auto-removed after upload (nothing reads it afterward; training consumes the mlflow run artifacts via runs:/ URIs). Co-Authored-By: Claude Opus 4.8 --- preprocessing/embeddings.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/preprocessing/embeddings.py b/preprocessing/embeddings.py index fe9c202..520f309 100644 --- a/preprocessing/embeddings.py +++ b/preprocessing/embeddings.py @@ -1,5 +1,5 @@ import asyncio -import shutil +import tempfile import time from pathlib import Path from typing import Any @@ -221,20 +221,24 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: max_concurrency=config.concurrency, ) - split_dir = Path(config.output_dir) / str(name) - split_dir.mkdir(parents=True, exist_ok=True) - tiles_parquet_dir = split_dir / "tiles" - if tiles_parquet_dir.exists(): - shutil.rmtree(tiles_parquet_dir) + # Isolated per-run staging dir: avoids cross-model contamination and + # stale-file carryover from concurrent or previous runs sharing + # output_dir. Auto-removed after upload; nothing reads it afterward + # (training reads the mlflow run artifacts via runs:/ URIs). + Path(config.output_dir).mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=config.output_dir) as tmp_root: + split_dir = Path(tmp_root) / str(name) + tiles_parquet_dir = split_dir / "tiles" + split_dir.mkdir(parents=True, exist_ok=True) - slides.to_parquet(split_dir / "slides.parquet", index=False) + slides.to_parquet(split_dir / "slides.parquet", index=False) - t = time.monotonic() - print(f"[main] starting write_parquet for split={name}") - ds.write_parquet(str(tiles_parquet_dir), min_rows_per_file=config.rows_per_file) - print(f"[main] write_parquet finished in {time.monotonic() - t:.1f}s") + t = time.monotonic() + print(f"[main] starting write_parquet for split={name}") + ds.write_parquet(str(tiles_parquet_dir), min_rows_per_file=config.rows_per_file) + print(f"[main] write_parquet finished in {time.monotonic() - t:.1f}s") - logger.log_artifacts(str(split_dir), str(name)) + logger.log_artifacts(str(split_dir), str(name)) if __name__ == "__main__": From 4167bb4f9737bb00767d0e71de4c4d635c4da627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vojt=C4=9Bch=20C=C3=ADfka?= <550433@mail.muni.cz> Date: Sun, 12 Jul 2026 22:19:57 +0200 Subject: [PATCH 2/2] fix: format --- preprocessing/embeddings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/preprocessing/embeddings.py b/preprocessing/embeddings.py index 520f309..fbbf6fb 100644 --- a/preprocessing/embeddings.py +++ b/preprocessing/embeddings.py @@ -235,7 +235,9 @@ def main(config: DictConfig, logger: MLFlowLogger) -> None: t = time.monotonic() print(f"[main] starting write_parquet for split={name}") - ds.write_parquet(str(tiles_parquet_dir), min_rows_per_file=config.rows_per_file) + ds.write_parquet( + str(tiles_parquet_dir), min_rows_per_file=config.rows_per_file + ) print(f"[main] write_parquet finished in {time.monotonic() - t:.1f}s") logger.log_artifacts(str(split_dir), str(name))