Fixes

scripts/create_resources/combine/process_datasets_xenium_nebius.sh

@@ -129,16 +129,6 @@ param_list:
     dataset_description: "Xenium V1 FFPE Human Breast IDC + 2021 Wu scRNAseq"
     dataset_organism: "homo_sapiens"
 
-  - id: "2026_10x_human_breast_cancer_atera_combined"
-    input_sp: "$input_dir/10x_atera/2026_10x_human_breast_cancer_atera/dataset.zarr"
-    input_sc: "$input_dir/wu_human_breast_cancer_sc/2021Wu_human_breast_cancer_sc/dataset.h5ad"
-    dataset_id: "2026_10x_human_breast_cancer_atera_combined"
-    dataset_name: "Human breast cancer combined 2026 10x Atera WTA 2021 Wu scRNAseq"
-    dataset_url: "https://www.10xgenomics.com/datasets/atera-wta-ffpe-human-breast-cancer"
-    dataset_reference: "https://doi.org/10.1038/s41588-021-00911-1"
-    dataset_summary: "Atera WTA FFPE Human Breast Cancer (DCIS Grade 3) + 2021 Wu scRNAseq"
-    dataset_description: "Atera WTA FFPE Human Breast Cancer (DCIS Grade 3) + 2021 Wu scRNAseq"
-    dataset_organism: "homo_sapiens"
 
 output_sc: "\$id/output_sc.h5ad"
 output_sp: "\$id/output_sp.zarr"

scripts/run_benchmark/cellposev4_params.yaml

@@ -0,0 +1,47 @@
+# Parameter sweep for the cellposev4 (Cellpose 4 / Cellpose-SAM) segmentation method.
+# Shared source of truth for both run_test_cellposev4_local.sh (read as a local file)
+# and run_test_cellposev4_nebius.sh (read from GitHub via a raw URL, since the Nebius
+# compute env pulls the repo but cannot see the launch host's local files).
+#
+# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting
+# (src/workflows/run_benchmark/main.nf). For every method the workflow builds:
+#   * one "default" variant using the `default:` args below, and
+#   * one extra variant per value in each `sweep:` list, with that ONE arg overridden.
+# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not
+# a full grid), so total cellposev4 variants = 1 default + sum(sweep list lengths) = 19.
+#
+# See src/methods_segmentation/cellposev4/NOTES.md ("Optimization / tuning") for the
+# rationale, tiers, and defaults. Trim or widen the lists to taste.
+parameters:
+  cellposev4:
+    # Baseline == the component's shipped (deliberately speed-tuned) defaults.
+    default:
+      diameter: 30.0
+      flow_threshold: 0.0
+      cellprob_threshold: 0.0
+      niter: 10
+      min_size: -1
+      resample: false
+    sweep:
+      # ---- Tier 1: highest impact on segmentation quality ----
+      # diameter: cellpose rescales so objects land near ~30 px (training mean),
+      # so 30.0 == "no rescaling". Xenium morphology (~0.2125 um/px) => nuclei
+      # ~35-40 px, whole cells ~60-70 px. Span small nuclei -> whole cells.
+      diameter: [15.0, 20.0, 40.0, 60.0]
+      # cellprob_threshold: recall<->precision dial (network logit ~ -6..+6, def 0).
+      # Lower -> recover more/dimmer cells; higher -> drop dim-region detections.
+      cellprob_threshold: [-3.0, -1.0, 1.0, 3.0]
+      # flow_threshold: 0.0 (our default) disables flow-shape QC (keeps ill-shaped
+      # masks); >0 re-enables it, more permissive as it rises. 0.4 = Cellpose default.
+      flow_threshold: [0.4, 0.6, 0.8]
+      # ---- Tier 2: quality/speed trade-offs ----
+      # niter: flow-dynamics iterations; 10 (default) is aggressively low and may
+      # under-converge large/elongated cells. Cellpose's own default is ~diameter-
+      # proportional (often ~200).
+      niter: [50, 100, 200]
+      # resample: BOOLEAN, default false -> the only meaningful non-default value is
+      # true (run dynamics at full resolution: smoother boundaries, slower).
+      resample: [true]
+      # min_size: -1 (default) disables small-mask removal; positive values drop
+      # progressively larger specks/debris.
+      min_size: [15, 30, 50]

src/datasets/loaders/bruker_cosmx/config.vsh.yaml

@@ -4,10 +4,15 @@ namespace: datasets/loaders
 argument_groups:
   - name: Inputs
     arguments:
-      - type: file
+      - type: string
         name: --input_raw
         example: "https://smi-public.objects.liquidweb.services/HalfBrain.zip"
-        description: "Download file url for the raw data"
+        description: |
+          URL/path to the raw data zip. Passed as a string (not a staged file) on
+          purpose: the script streams it in place and extracts only the ~15 GB sopa
+          needs (Morphology2D + CellLabels), skipping the ~190 GB of Morphology3D /
+          AnalysisResults. Staging the full zip (175 GiB) alongside the extraction
+          overruns the node's scratch and OOMs. Supports s3:// (via s3fs) or a local path.
       - type: file
         name: --input_flat_files
         example: "https://smi-public.objects.liquidweb.services/Half%20%20Brain%20simple%20%20files%20.zip"
@@ -65,9 +70,10 @@ engines:
     __merge__: 
       - /src/base/setup_spatialdata_partial.yaml
     setup:
-      - type: python 
+      - type: python
         pypi:
           - sopa
+          - s3fs  # stream the raw zip from s3:// and extract only what sopa needs (see --input_raw)
   - type: native
 
 runners:

src/datasets/loaders/bruker_cosmx/script.py

@@ -118,39 +118,73 @@ def log(msg):
 TMP_DIR = Path(meta["temp_dir"] or "/tmp")
 TMP_DIR.mkdir(parents=True, exist_ok=True)
 
-def extract_zip(input_zip: Path, output_dir: Path, strip_root: bool = False):
+# CosMx export directories that sopa's cosmx reader never touches. Skipping them on
+# extraction is what makes the mouse-brain hemisphere fit in scratch: the raw zip is
+# ~208 GB, but ~190 GB of that is Morphology3D (per-z-plane 3D stacks — sopa uses the
+# 2D Morphology2D composite) plus AnalysisResults (decoding CSVs) and RunSummary (logs).
+# What's left (Morphology2D + FOV*/CellLabels + the flat CSVs) is ~15 GB. Extracting the
+# whole thing next to the staged zip overruns the node's ~220 GB scratch and OOMs (137).
+SKIP_DIRS = {"Morphology3D", "AnalysisResults", "RunSummary"}
+
+def _member_wanted(name: str) -> bool:
+    parts = name.split("/")
+    if name.startswith("__MACOSX/") or parts[-1] == ".DS_Store":
+        return False
+    return not any(part in SKIP_DIRS for part in parts)
+
+def _open_zip_source(src):
+    """Seekable binary handle for a local path or an s3:// URL (streamed, never fully downloaded)."""
+    s = str(src)
+    if s.startswith("s3://"):
+        import s3fs
+        # openproblems-data is public; anonymous read is deterministic and needs no creds.
+        fs = s3fs.S3FileSystem(anon=True, default_block_size=32 * 2**20)
+        return fs.open(s, "rb")
+    return open(s, "rb")
+
+def extract_zip(input_zip, output_dir: Path, strip_root: bool = False):
     """
-    Extracts the input zip file to the specified output directory.
+    Stream a zip (local path or s3:// URL) into output_dir, skipping the CosMx
+    directories sopa never reads (see SKIP_DIRS). Only the wanted members are
+    transferred, so a remote zip streams just the bytes we keep — it is never
+    downloaded in full first.
 
     Arguments:
-    
-    - input_zip: Path to the input zip file.
-    - output_dir: Path to the directory where the contents of the zip file will be extracted
-    - strip_root: If True, and if all entries in the zip file share exactly one top-level directory, then this top-level directory will be stripped when extracting. 
+
+    - input_zip: local path or s3:// URL of the input zip.
+    - output_dir: directory where the (filtered) contents are written.
+    - strip_root: if True and all entries share exactly one top-level directory, strip it.
     """
     output_dir = Path(output_dir)
-    with zipfile.ZipFile(input_zip, 'r') as zip_ref:
+    kept = skipped = 0
+    kept_bytes = 0
+    with _open_zip_source(input_zip) as fh, zipfile.ZipFile(fh, "r") as zip_ref:
         members = zip_ref.infolist()
 
         # only strip when all entries share exactly one top-level directory
         roots = {Path(m.filename).parts[0] for m in members if m.filename.strip("/") and not m.filename.startswith("__MACOSX/")}
-        if not (strip_root and len(roots) == 1):
-            zip_ref.extractall(output_dir)
-            return
+        do_strip = strip_root and len(roots) == 1
 
         for member in members:
-            if member.filename.startswith("__MACOSX/"):
-                continue  # skip macOS resource-fork entries, don't recreate them
-            parts = Path(member.filename).parts[1:]
+            if not _member_wanted(member.filename):
+                skipped += 1
+                continue
+            parts = Path(member.filename).parts
+            if do_strip:
+                parts = parts[1:]
             if not parts:
                 continue  # the wrapper directory entry itself
             target = output_dir.joinpath(*parts)
             if member.is_dir():
                 target.mkdir(parents=True, exist_ok=True)
-            else:
-                target.parent.mkdir(parents=True, exist_ok=True)
-                with zip_ref.open(member) as src, open(target, "wb") as dst:
-                    shutil.copyfileobj(src, dst)
+                continue
+            target.parent.mkdir(parents=True, exist_ok=True)
+            with zip_ref.open(member) as src, open(target, "wb") as dst:
+                shutil.copyfileobj(src, dst)
+            kept += 1
+            kept_bytes += member.file_size
+    log(f"Extracted {kept} files ({kept_bytes / 1e9:.1f} GB); skipped {skipped} members "
+        f"(dirs {sorted(SKIP_DIRS)} + macOS cruft)")
 
 # Extract zip files
 log("Extract zip of raw files")
@@ -199,6 +233,59 @@ def extract_zip(input_zip: Path, output_dir: Path, strip_root: bool = False):
 else:
     raise AssertionError(f"No CellLabels folder and no per-FOV CellLabels tifs found in {DATA_DIR}")
 
+#############################################
+# Memory optimization: lean transcript read #
+#############################################
+# sopa's CosMx reader loads the ENTIRE `*_tx_file.csv` into a pandas DataFrame in
+# one `pd.read_csv` (sopa/io/reader/cosmx.py::_CosMXReader.read_transcripts). For
+# the mouse-brain half-hemisphere that's hundreds of millions of rows, and the
+# string columns ('target', 'CellComp', 'cell', ...) come in as object dtype
+# (~60 B/row) — this is the peak that blows past even the 480 GB veryhighmem cap.
+#
+# We wrap the `read_csv` used inside sopa's cosmx module so that, *only for the
+# transcripts file*, we (a) read the 'target' feature column as categorical and
+# (b) keep just the columns sopa's stitched-FOV path (read_transcripts +
+# _get_global_cell_id) and the downstream transcript schema actually use. Every
+# other sopa read (fov positions, metadata, counts, polygons) is left untouched,
+# and all of sopa's coordinate/stitching math runs unchanged on the leaner frame.
+from sopa.io.reader import cosmx as _cosmx_mod
+
+_real_pd = _cosmx_mod.pd
+_real_read_csv = _real_pd.read_csv
+
+# Columns used by read_transcripts (fov=None) + _get_global_cell_id, plus 'z'
+# which the downstream transcript schema allows. Everything else in the tx file
+# (x_local_px, y_local_px, cell, CellComp, ...) is unused downstream.
+_TX_KEEP = ["fov", "cell_ID", "target", "x_global_px", "y_global_px", "z"]
+_TX_REQUIRED = {"target", "x_global_px", "y_global_px", "fov", "cell_ID"}
+
+def _lean_read_csv(filepath_or_buffer, *args, **kwargs):
+    name = str(filepath_or_buffer)
+    if "_tx_file.csv" in name and "usecols" not in kwargs:
+        header = _real_read_csv(
+            filepath_or_buffer, nrows=0,
+            compression=kwargs.get("compression", "infer"),
+        )
+        cols = set(header.columns)
+        if _TX_REQUIRED.issubset(cols):
+            kwargs["usecols"] = [c for c in _TX_KEEP if c in cols]
+            dtype = dict(kwargs.get("dtype") or {})
+            dtype.setdefault("target", "category")
+            kwargs["dtype"] = dtype
+            log(f"Lean transcript read of {Path(name).name}: usecols={kwargs['usecols']}, target=category")
+    return _real_read_csv(filepath_or_buffer, *args, **kwargs)
+
+class _PandasReadCsvProxy:
+    """Forwards every attribute to pandas, but leans the transcripts read_csv."""
+    def __init__(self, real):
+        self.__dict__["_real"] = real
+    def read_csv(self, *args, **kwargs):
+        return _lean_read_csv(*args, **kwargs)
+    def __getattr__(self, item):
+        return getattr(self._real, item)
+
+_cosmx_mod.pd = _PandasReadCsvProxy(_real_pd)
+
 #########################################
 # Convert raw files to spatialdata zarr #
 #########################################

src/datasets/workflows/process_bruker_cosmx/config.vsh.yaml

@@ -4,10 +4,10 @@ namespace: datasets/workflows
 argument_groups:
   - name: Inputs
     arguments:
-      - type: file
+      - type: string
         name: --input_raw
         example: "https://smi-public.objects.liquidweb.services/HalfBrain.zip"
-        description: "Download file url for the raw data"
+        description: "URL/path to the raw data zip. A string (not a staged file) so the loader can stream it and extract only what sopa needs — see the loader's --input_raw."
       - type: file
         name: --input_flat_files
         example: "https://smi-public.objects.liquidweb.services/Half%20%20Brain%20simple%20%20files%20.zip"

src/methods_cell_type_annotation/rctd/script.R

@@ -18,7 +18,7 @@ par <- list(
 )
 
 meta <- list(
-  'cpus': 4,
+  cpus = 4
 )
 
 ## VIASH END

src/methods_segmentation/cellposev4/config.vsh.yaml

@@ -12,7 +12,11 @@ links:
   documentation: "https://cellpose.readthedocs.io/en/latest/"
   repository: "https://github.com/mouseland/cellpose"
 references:
-  doi: "10.1038/s41592-020-01018-x"
+  doi:
+    # Cellpose-SAM (cpsam): the model this component actually runs
+    - "10.1101/2025.04.28.651001"
+    # Original Cellpose: the flow-dynamics framework it builds on
+    - "10.1038/s41592-020-01018-x"
 
 __merge__: /src/api/comp_method_segmentation.yaml
 
@@ -25,6 +29,10 @@ arguments:
     type: double
     default: 0.0
     description: "Flow error threshold. Set to 0 to skip flow quality check for faster execution."
+  - name: --cellprob_threshold
+    type: double
+    default: 0.0
+    description: "Cell probability threshold (network output ~-6 to 6). Pixels above it are used to seed masks. Lower to recover more/dimmer cells; raise to drop detections in dim/low-contrast regions."
   - name: --niter
     type: integer
     default: 10

src/methods_segmentation/cellposev4/script.py

@@ -18,6 +18,7 @@
   "output": "segmentation.zarr",
   "diameter": 30.0,
   "flow_threshold": 0.0,
+  "cellprob_threshold": 0.0,
   "niter": 10,
   "min_size": -1,
   "resample": False,
@@ -50,7 +51,7 @@ def convert_to_lower_dtype(arr):
 print('Initializing Cellpose model', flush=True)
 model = CellposeModel(gpu=torch.cuda.is_available())
 
-eval_params = {k: par[k] for k in ("diameter", "flow_threshold", "niter", "min_size", "resample") if par.get(k) is not None}
+eval_params = {k: par[k] for k in ("diameter", "flow_threshold", "cellprob_threshold", "niter", "min_size", "resample") if par.get(k) is not None}
 print('Running Cellpose segmentation with parameters:', eval_params, flush=True)
 masks, _, _ = model.eval(image[0], progress=True, **eval_params)
 

src/methods_transcript_assignment/comseg/script.py

@@ -37,22 +37,37 @@
 
 # Read input files
 print('Reading input files', flush=True)
-sdata = sd.read_zarr(par['input_ist'])
+sdata_src = sd.read_zarr(par['input_ist'])
 sdata_segm = sd.read_zarr(par['input_segmentation'])
 
 # Multi-partition parquet restarts its index at 0 per partition, producing duplicate
 # index labels that break sopa's cell_id assignment and the final write with
 # "cannot reindex on an axis with duplicate labels". Rebuild the transcripts as a
 # single-partition frame with a clean RangeIndex before any sopa op touches them.
-_tx = sdata[par['transcripts_key']]
+_tx = sdata_src[par['transcripts_key']]
 _tx_reset = dd.from_pandas(_tx.compute().reset_index(drop=True), npartitions=1)
 _tx_reset.attrs.update(_tx.attrs)
-del sdata[par['transcripts_key']]
-sdata[par['transcripts_key']] = _tx_reset
 
 # Convert the prior segmentation to polygons
-sdata["segmentation_boundaries"] = sd.to_polygons(sdata_segm["segmentation"])
-del sdata["segmentation_boundaries"]["label"] # make_transcript_patches will create a new label column and fails if one exists.
+segmentation_boundaries = sd.to_polygons(sdata_segm["segmentation"])
+del segmentation_boundaries["label"]  # make_transcript_patches will create a new label column and fails if one exists.
+
+# Run sopa/ComSeg on a FRESH, in-memory SpatialData. sd.read_zarr() leaves `sdata_src`
+# backed by the shared source zarr (sdata_src.path -> input_ist), and sopa's
+# make_image_patches / make_transcript_patches / segmentation.comseg all write their new
+# elements (image_patches, transcripts_patches, comseg_boundaries) and a .sopa_cache back
+# through that path -- mutating the supposed-to-be-immutable source dataset in place. If a
+# concurrent run or an OOM kill interrupts one of those writes, it leaves a half-written
+# element (e.g. shapes/image_patches with no shapes.parquet) that then breaks EVERY
+# downstream reader of the dataset. baysor and proseg avoid this by running sopa on a fresh
+# in-memory object; do the same here (carrying the image for make_image_patches and the
+# prior boundaries for the transcript patches) so all sopa writes land in the ephemeral
+# work dir, never the source zarr.
+sdata = sd.SpatialData(
+    images={"image": sdata_src["image"]},
+    points={par['transcripts_key']: _tx_reset},
+    shapes={"segmentation_boundaries": segmentation_boundaries},
+)
 
 # Make patches
 sopa.make_image_patches(sdata, image_key="image", patch_width=par["patch_width"], patch_overlap=par["patch_overlap"])

src/methods_transcript_assignment/pciseq/script.py

@@ -131,10 +131,25 @@ def eta_update_no_assert(self):
 
 #same as before
 if isinstance(sdata_segm["segmentation"], xr.DataTree):
-    label_image = sdata_segm["segmentation"]["scale0"].image.to_numpy() 
+    label_image = sdata_segm["segmentation"]["scale0"].image.to_numpy()
 else:
      label_image = sdata_segm["segmentation"].to_numpy()
 
+# Guard against an empty segmentation. pciSeq builds `coo = coo_matrix(label_image)` and
+# then calls `coo.data.max()` in stage_data; when the label image has no cells (all zeros)
+# `coo.data` is a zero-size array and numpy raises the cryptic
+# "zero-size array to reduction operation maximum which has no identity".
+# Fail fast with an actionable message instead. This happens when the upstream `cell_labels`
+# copied by custom_segmentation is empty (observed for the vizgen lung-cancer merscope dataset,
+# whose polygon->label rasterization produced an all-zero image).
+if not (label_image > 0).any():
+    raise ValueError(
+        f"Segmentation '{par['input_segmentation']}' contains no cells: the label image "
+        f"(shape={label_image.shape}, dtype={label_image.dtype}) is entirely zero. pciSeq "
+        f"cannot assign transcripts without any cell labels. Check the upstream segmentation "
+        f"(for custom_segmentation this means the dataset's `cell_labels` is empty)."
+    )
+
 # pciSeq internally drops spots that fall OUTSIDE the label image, and returns
 # per-spot assignments only for the kept spots; txsim.run_pciSeq then does
 # `spots['cell'] = assignments`, which raises "Length of values ... does not match