From 418a3405fa2ab4a051e5337ee3188b37bd6a286e Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 14:00:23 +0200
Subject: [PATCH 01/14] fix novel_predict's removed_vars check

`removed_vars` round-trips through h5ad as an array, so the plain truthiness
test raised "The truth value of an array with more than one element is
ambiguous" as soon as training dropped more than one all-zero feature -- which
is every dataset since #33.
---
 src/methods/novel/novel_predict/script.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/methods/novel/novel_predict/script.py b/src/methods/novel/novel_predict/script.py
index ddc0233c..7518a167 100644
--- a/src/methods/novel/novel_predict/script.py
+++ b/src/methods/novel/novel_predict/script.py
@@ -48,8 +48,8 @@
 input_test_mod1.X = input_test_mod1.layers['normalized'].tocsr()
 
 # Remove vars that were removed from training set. Mostly only applicable for testing.
-if input_train_mod2.uns.get("removed_vars"):
-  rem_var = input_train_mod2.uns["removed_vars"]
+rem_var = input_train_mod2.uns.get("removed_vars")
+if rem_var is not None and len(rem_var) > 0:
   input_test_mod1 = input_test_mod1[:, ~input_test_mod1.var_names.isin(rem_var)]
 
 del input_train_mod2

From be651157031efa62909fb2ffc9e0703c1d7a981b Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 14:03:39 +0200
Subject: [PATCH 02/14] exit 99 for the modality pairs babel and scbutterfly do
 not support to avoid unnecessary retries

---
 src/methods/babel/babel_train/config.vsh.yaml               | 1 +
 src/methods/babel/babel_train/script.py                     | 6 ++++--
 src/methods/scbutterfly/butterfly_common.py                 | 6 ++++--
 src/methods/scbutterfly/scbutterfly_predict/config.vsh.yaml | 1 +
 src/methods/scbutterfly/scbutterfly_train/config.vsh.yaml   | 1 +
 5 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/methods/babel/babel_train/config.vsh.yaml b/src/methods/babel/babel_train/config.vsh.yaml
index e8bead66..b5c2c531 100644
--- a/src/methods/babel/babel_train/config.vsh.yaml
+++ b/src/methods/babel/babel_train/config.vsh.yaml
@@ -16,6 +16,7 @@ resources:
     dest: losses.py
   - path: ../chrom_utils.py
     dest: chrom_utils.py
+  - path: /src/utils/exit_codes.py
 test_resources:
   - path: /resources_test/task_predict_modality/openproblems_neurips2021/bmmc_multiome/swap
     dest: resources_test/task_predict_modality/openproblems_neurips2021/bmmc_multiome/swap
diff --git a/src/methods/babel/babel_train/script.py b/src/methods/babel/babel_train/script.py
index f35cedfe..e7011046 100644
--- a/src/methods/babel/babel_train/script.py
+++ b/src/methods/babel/babel_train/script.py
@@ -2,6 +2,8 @@
 import pickle
 import sys
 
+from exit_codes import exit_non_applicable
+
 import anndata as ad
 import numpy as np
 import scanpy as sc
@@ -126,14 +128,14 @@ def get_loss(self, y_pred, y_true, X=None, training=False):
 modality2 = adata_mod2_train.uns.get("modality")
 
 if {modality1, modality2} != {"GEX", "ATAC"}:
-    raise ValueError(
+    exit_non_applicable(
         f"babel only supports GEX<->ATAC translation, got modalities "
         f"({modality1!r}, {modality2!r}). This is a direct port of BABEL "
         "(Wu et al. 2021), whose architecture is not adapted for other modality pairs."
     )
 
 if modality1 != "ATAC":
-    raise ValueError(
+    exit_non_applicable(
         f"babel only supports ATAC->GEX prediction (input_train_mod1 must be ATAC, "
         f"input_train_mod2 must be GEX), got mod1={modality1!r}, mod2={modality2!r}. "
         "The GEX->ATAC direction of this BABEL port collapses to a per-peak base-rate "
diff --git a/src/methods/scbutterfly/butterfly_common.py b/src/methods/scbutterfly/butterfly_common.py
index 4ddf855b..dd63e3c9 100644
--- a/src/methods/scbutterfly/butterfly_common.py
+++ b/src/methods/scbutterfly/butterfly_common.py
@@ -9,6 +9,8 @@
 
 import logging
 
+from exit_codes import exit_non_applicable
+
 import anndata as ad
 import numpy as np
 import scanpy as sc
@@ -58,11 +60,11 @@ def _safe_bce(input, target, *args, **kwargs):
 
 
 def detect_direction(train_mod1, train_mod2):
-    """Return ('GEX2ATAC'|'ATAC2GEX', mod1, mod2), raising on non-Multiome data."""
+    """Return ('GEX2ATAC'|'ATAC2GEX', mod1, mod2), exiting 99 on non-Multiome data."""
     mod1 = train_mod1.uns["modality"]
     mod2 = train_mod2.uns["modality"]
     if {mod1, mod2} != {"GEX", "ATAC"}:
-        raise ValueError(
+        exit_non_applicable(
             f"scbutterfly only supports Multiome GEX<->ATAC, got mod1={mod1}, mod2={mod2}"
         )
     direction = "GEX2ATAC" if mod2 == "ATAC" else "ATAC2GEX"
diff --git a/src/methods/scbutterfly/scbutterfly_predict/config.vsh.yaml b/src/methods/scbutterfly/scbutterfly_predict/config.vsh.yaml
index 595b4e3d..ae9aac7b 100644
--- a/src/methods/scbutterfly/scbutterfly_predict/config.vsh.yaml
+++ b/src/methods/scbutterfly/scbutterfly_predict/config.vsh.yaml
@@ -12,6 +12,7 @@ resources:
     type: python_script
   - path: ../butterfly_common.py
   - path: ../chrom_utils.py
+  - path: /src/utils/exit_codes.py
 test_resources:
   - path: /resources_test/task_predict_modality/openproblems_neurips2021/bmmc_multiome/swap
     dest: resources_test/task_predict_modality/openproblems_neurips2021/bmmc_multiome/swap
diff --git a/src/methods/scbutterfly/scbutterfly_train/config.vsh.yaml b/src/methods/scbutterfly/scbutterfly_train/config.vsh.yaml
index 87a917c6..b68ca593 100644
--- a/src/methods/scbutterfly/scbutterfly_train/config.vsh.yaml
+++ b/src/methods/scbutterfly/scbutterfly_train/config.vsh.yaml
@@ -16,6 +16,7 @@ resources:
     type: python_script
   - path: ../butterfly_common.py
   - path: ../chrom_utils.py
+  - path: /src/utils/exit_codes.py
 test_resources:
   - path: /resources_test/task_predict_modality/openproblems_neurips2021/bmmc_multiome/swap
     dest: resources_test/task_predict_modality/openproblems_neurips2021/bmmc_multiome/swap

From 2fed84e159d220c65572397d87bbe29c52b2faf3 Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 14:11:07 +0200
Subject: [PATCH 03/14] run senkin_tmp_train on the tensorflow base image so it
 can use a gpu

---
 src/methods/senkin_tmp/senkin_tmp_train/config.vsh.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/methods/senkin_tmp/senkin_tmp_train/config.vsh.yaml b/src/methods/senkin_tmp/senkin_tmp_train/config.vsh.yaml
index dec48570..1f228bb2 100644
--- a/src/methods/senkin_tmp/senkin_tmp_train/config.vsh.yaml
+++ b/src/methods/senkin_tmp/senkin_tmp_train/config.vsh.yaml
@@ -32,14 +32,13 @@ arguments:
     description: TSVD components for reducing LightGBM predictions before NN input.
 engines:
   - type: docker
-    image: openproblems/base_pytorch_nvidia:1
+    image: openproblems/base_tensorflow_nvidia:1
     setup:
       - type: docker
         run: pip install --no-cache-dir --no-deps git+https://github.com/lueckenlab/senkin-tmp-cite-pred.git
       - type: python
         packages:
           - lightgbm>=4.0
-          - tensorflow>=2.12
           - scikit-learn>=1.1
           - mudata>=0.2
           - muon>=0.1
@@ -48,4 +47,4 @@ runners:
   - type: executable
   - type: nextflow
     directives:
-      label: [highmem, hightime, midcpu]
+      label: [highmem, hightime, midcpu, gpu]

From 6ced88d077e1b339d0677c8a013c1984fd6ac3ef Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 14:14:39 +0200
Subject: [PATCH 04/14] score non-finite predictions as zero instead of halting
 the metric

---
 src/metrics/correlation/script.R | 12 ++++++++----
 src/metrics/mse/script.py        | 11 ++++++++++-
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/metrics/correlation/script.R b/src/metrics/correlation/script.R
index 79cbbd3d..977a6fdd 100644
--- a/src/metrics/correlation/script.R
+++ b/src/metrics/correlation/script.R
@@ -33,6 +33,14 @@ cat("Computing correlation metrics\n")
 tv <- ad_sol$layers[["normalized"]]
 pv <- ad_pred$layers[["normalized"]]
 
+# score non-finite predictions as zero, like the zero-variance case below
+pv_values <- if (methods::is(pv, "sparseMatrix")) pv@x else pv
+non_finite <- !is.finite(pv_values)
+if (any(non_finite)) {
+  cat("Prediction contains", sum(non_finite), "non-finite values, scoring them as 0\n")
+  if (methods::is(pv, "sparseMatrix")) pv@x[non_finite] <- 0 else pv[non_finite] <- 0
+}
+
 # precompute sds
 tv_sd2 <- proxyC::colSds(tv)
 pv_sd2 <- proxyC::colSds(pv)
@@ -65,8 +73,6 @@ spearman_vec_1 <- paired_similarity(tv, pv, margin = 1, method = "spearman")
 
 pearson_vec_1[tv_sd1 == 0 | pv_sd1 == 0] <- 0
 spearman_vec_1[tv_sd1 == 0 | pv_sd1 == 0] <- 0
-# pearson_vec_1[!is.finite(pearson_vec_1) | pearson_vec_1 > 10] <- 0
-# spearman_vec_1[!is.finite(spearman_vec_1) | spearman_vec_1 > 10] <- 0
 
 mean_pearson_per_cell <- mean(pearson_vec_1)
 mean_spearman_per_cell <- mean(spearman_vec_1)
@@ -76,8 +82,6 @@ spearman_vec_2 <- paired_similarity(tv, pv, margin = 2, method = "spearman")
 
 pearson_vec_2[tv_sd2 == 0 | pv_sd2 == 0] <- 0
 spearman_vec_2[tv_sd2 == 0 | pv_sd2 == 0] <- 0
-# pearson_vec_2[!is.finite(pearson_vec_2) | pearson_vec_2 > 10] <- 0
-# spearman_vec_2[!is.finite(spearman_vec_2) | spearman_vec_2 > 10] <- 0
 
 mean_pearson_per_gene <- mean(pearson_vec_2)
 mean_spearman_per_gene <- mean(spearman_vec_2)
diff --git a/src/metrics/mse/script.py b/src/metrics/mse/script.py
index 703f6226..1efc14b1 100644
--- a/src/metrics/mse/script.py
+++ b/src/metrics/mse/script.py
@@ -27,7 +27,16 @@
 logging.info("Computing MSE metrics")
 
 # coerce to sparse -- sparse minus dense yields a np.matrix, which has no .power()
-tmp = csr_matrix(ad_sol.layers["normalized"]) - csr_matrix(ad_pred.layers["normalized"])
+sol = csr_matrix(ad_sol.layers["normalized"])
+pred = csr_matrix(ad_pred.layers["normalized"])
+
+# score non-finite predictions as zero, rather than returning a NaN metric
+non_finite = ~np.isfinite(pred.data)
+if non_finite.any():
+  logging.info("Prediction contains %d non-finite values, scoring them as 0", non_finite.sum())
+  pred.data[non_finite] = 0
+
+tmp = sol - pred
 rmse = np.sqrt(tmp.power(2).mean())
 mae = np.abs(tmp).mean()
 

From 694d878bd7e0d274d5cb9a6e456cca15e3839ef9 Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 14:21:03 +0200
Subject: [PATCH 05/14] densify ss_opm's per-cell statistics one row block at a
 time

---
 src/methods/ss_opm/ss_opm_common.py | 31 ++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/src/methods/ss_opm/ss_opm_common.py b/src/methods/ss_opm/ss_opm_common.py
index 416853b2..920b1c32 100644
--- a/src/methods/ss_opm/ss_opm_common.py
+++ b/src/methods/ss_opm/ss_opm_common.py
@@ -4,6 +4,9 @@
 import pandas as pd
 import scipy.sparse
 
+# cells per block when densifying the expression matrix for per-cell statistics
+ROW_BLOCK = 1000
+
 # the cell types the original ss_opm model was trained against. only used to name the
 # cell_ratio_* columns it expects; the ratios themselves are derived from the data when
 # cell type labels are available.
@@ -66,16 +69,26 @@ def build_metadata(
     obs["batch"] = adata.obs["batch"].values
     obs["day"] = extract_day(adata.obs["batch"], day_pattern).fillna(0).values
 
-    # per-cell statistics from the normalized expression layer
+    # per-cell statistics from the normalized expression layer, densified one row
+    # block at a time -- the whole matrix is 222 GiB on the multiome datasets
     X = adata.layers["normalized"]
-    X_dense = X.toarray() if scipy.sparse.issparse(X) else np.asarray(X, dtype=float)
-
-    obs["nonzero_ratio"] = (X_dense != 0).mean(axis=1)
-    obs["nonzero_q25"] = np.percentile(X_dense, 25, axis=1)
-    obs["nonzero_q50"] = np.percentile(X_dense, 50, axis=1)
-    obs["nonzero_q75"] = np.percentile(X_dense, 75, axis=1)
-    obs["mean"] = X_dense.mean(axis=1)
-    obs["std"] = X_dense.std(axis=1)
+    names = ("nonzero_ratio", "nonzero_q25", "nonzero_q50", "nonzero_q75", "mean", "std")
+    stats = {name: np.empty(adata.n_obs, dtype=float) for name in names}
+
+    for start in range(0, adata.n_obs, ROW_BLOCK):
+        end = min(start + ROW_BLOCK, adata.n_obs)
+        block = X[start:end]
+        block = block.toarray() if scipy.sparse.issparse(block) else np.asarray(block, dtype=float)
+
+        stats["nonzero_ratio"][start:end] = (block != 0).mean(axis=1)
+        stats["nonzero_q25"][start:end] = np.percentile(block, 25, axis=1)
+        stats["nonzero_q50"][start:end] = np.percentile(block, 50, axis=1)
+        stats["nonzero_q75"][start:end] = np.percentile(block, 75, axis=1)
+        stats["mean"][start:end] = block.mean(axis=1)
+        stats["std"][start:end] = block.std(axis=1)
+
+    for name in names:
+        obs[name] = stats[name]
 
     if group_by_batch:
         batches = adata.obs["batch"].unique().tolist()

From 27205b5ce7e7a1d7e57c4af3c3896663d8f3129f Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 14:21:35 +0200
Subject: [PATCH 06/14] update changelog

---
 CHANGELOG.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0751137d..ee40f87b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -80,6 +80,16 @@
 
 * `novel_predict`: Move the model and the input batch onto the selected device. It picked `cuda:0` when a GPU was present but never used it, so the component requested a GPU node and ran inference on CPU (PR #54).
 
+* `correlation`, `mse`: Score non-finite predictions as zero instead of halting. `sd()` on a prediction holding `NaN` returns `NA`, so `correlation` died on `if (NA)` rather than scoring the method, and `mse` wrote a `NaN` score. A method emitting `NaN` should land at the bottom of the scale, not take the metric down with it (PR #59).
+
+* `babel_train`, `scbutterfly`: Exit 99 for the modality pairs they do not support, rather than raising a `ValueError`. Both are GEX<->ATAC only, so on the four CITE datasets they failed with exit 1 and were retried three times each (PR #59).
+
+* `novel_predict`: Test `uns["removed_vars"]` for length rather than truthiness. It round-trips through h5ad as an array, so the check raised "The truth value of an array with more than one element is ambiguous" as soon as training dropped more than one all-zero feature (PR #59).
+
+* `senkin_tmp_train`: Build on `openproblems/base_tensorflow_nvidia:1` and ask for a `gpu`. The pip-installed TensorFlow was built against CUDA 12 while `base_pytorch_nvidia` ships CUDA 13, so it silently trained on CPU and hit its walltime on every dataset (PR #59).
+
+* `ss_opm`: Compute the per-cell statistics in `build_metadata()` one row block at a time. Densifying the whole normalized layer asked for a 222 GiB allocation on the multiome datasets, which fits on no node (PR #59).
+
 # task_predict_modality 0.1.1
 
 ## NEW FUNCTIONALITY

From 929401a5225c35c8883ed86014dd23d22a857cd7 Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 16:09:37 +0200
Subject: [PATCH 07/14] import exit_codes after the resources dir is on
 sys.path

---
 src/methods/babel/babel_train/script.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/methods/babel/babel_train/script.py b/src/methods/babel/babel_train/script.py
index e7011046..a0576d87 100644
--- a/src/methods/babel/babel_train/script.py
+++ b/src/methods/babel/babel_train/script.py
@@ -2,8 +2,6 @@
 import pickle
 import sys
 
-from exit_codes import exit_non_applicable
-
 import anndata as ad
 import numpy as np
 import scanpy as sc
@@ -35,6 +33,7 @@
 sys.path.append(meta["resources_dir"])
 
 from chrom_utils import parse_chrom_groups
+from exit_codes import exit_non_applicable
 from model import AssymSplicedAutoEncoder
 from losses import QuadLoss
 

From dbb177758e972f61b2fbeb40964bf43625cd017b Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 17:38:18 +0200
Subject: [PATCH 08/14] build senkin_tmp_train on a cuda base so tensorflow
 2.20+ can reach the gpu

---
 .../senkin_tmp_train/config.vsh.yaml           | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/methods/senkin_tmp/senkin_tmp_train/config.vsh.yaml b/src/methods/senkin_tmp/senkin_tmp_train/config.vsh.yaml
index 1f228bb2..6f549b24 100644
--- a/src/methods/senkin_tmp/senkin_tmp_train/config.vsh.yaml
+++ b/src/methods/senkin_tmp/senkin_tmp_train/config.vsh.yaml
@@ -32,17 +32,31 @@ arguments:
     description: TSVD components for reducing LightGBM predictions before NN input.
 engines:
   - type: docker
-    image: openproblems/base_tensorflow_nvidia:1
+    image: nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04
     setup:
+      - type: apt
+        packages: [python3, python3-venv, procps, git, libgomp1]
       - type: docker
-        run: pip install --no-cache-dir --no-deps git+https://github.com/lueckenlab/senkin-tmp-cite-pred.git
+        run: python3 -m venv /opt/venv
+      - type: docker
+        env: [PATH=/opt/venv/bin:$PATH]
       - type: python
         packages:
+          - tensorflow[and-cuda]>=2.20
+          - anndata
+          - scanpy
+          - pyyaml
+          - requests
+          - jsonschema
           - lightgbm>=4.0
           - scikit-learn>=1.1
           - mudata>=0.2
           - muon>=0.1
           - fast-array-utils
+        github:
+          - openproblems-bio/core#subdirectory=packages/python/openproblems
+      - type: docker
+        run: pip install --no-cache-dir --no-deps git+https://github.com/lueckenlab/senkin-tmp-cite-pred.git
 runners:
   - type: executable
   - type: nextflow

From d319e4f85393eaff97bd6162cb1914e9fa209502 Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 17:38:43 +0200
Subject: [PATCH 09/14] update changelog

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ee40f87b..89cc2fe8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -86,7 +86,7 @@
 
 * `novel_predict`: Test `uns["removed_vars"]` for length rather than truthiness. It round-trips through h5ad as an array, so the check raised "The truth value of an array with more than one element is ambiguous" as soon as training dropped more than one all-zero feature (PR #59).
 
-* `senkin_tmp_train`: Build on `openproblems/base_tensorflow_nvidia:1` and ask for a `gpu`. The pip-installed TensorFlow was built against CUDA 12 while `base_pytorch_nvidia` ships CUDA 13, so it silently trained on CPU and hit its walltime on every dataset (PR #59).
+* `senkin_tmp_train`: Build on `nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04` with `tensorflow[and-cuda]`, and ask for a `gpu`. TensorFlow was installed on a PyTorch base shipping CUDA 13 while TF is built against CUDA 12, so it silently trained on CPU and hit its walltime on all eight datasets. `openproblems/base_tensorflow_nvidia:1` reaches the GPU but is frozen at TF 2.17, below the 2.20 the method requires -- nvcr no longer publishes TensorFlow images, so that base wants rebuilding on a CUDA image in openproblems-bio/core (PR #59).
 
 * `ss_opm`: Compute the per-cell statistics in `build_metadata()` one row block at a time. Densifying the whole normalized layer asked for a 222 GiB allocation on the multiome datasets, which fits on no node (PR #59).
 

From 55f1a71adc6676e87f131680ccb5c6e46b27e3f2 Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 21:22:24 +0200
Subject: [PATCH 10/14] apply ss_opm's safe normalizers in predict too, not
 just train

---
 src/methods/ss_opm/ss_opm_common.py         | 41 +++++++++++++++++++++
 src/methods/ss_opm/ss_opm_predict/script.py |  4 +-
 src/methods/ss_opm/ss_opm_train/script.py   | 38 ++-----------------
 3 files changed, 47 insertions(+), 36 deletions(-)

diff --git a/src/methods/ss_opm/ss_opm_common.py b/src/methods/ss_opm/ss_opm_common.py
index 920b1c32..3a55ed9a 100644
--- a/src/methods/ss_opm/ss_opm_common.py
+++ b/src/methods/ss_opm/ss_opm_common.py
@@ -120,3 +120,44 @@ def build_metadata(
             obs[f"batch_sv{i}"] = 0.0
 
     return obs
+
+
+def _safe_median_normalize(values, ignore_zero=True, log=False):
+    """Median-normalize rows, substituting 1 (identity) when the median is 0 or NaN."""
+    arr = np.asarray(values.toarray() if hasattr(values, 'toarray') else values, dtype=float).copy()
+    for_median = arr.copy()
+    if ignore_zero:
+        for_median[for_median == 0] = np.nan
+    med = np.nanquantile(for_median, q=0.5, axis=1)
+    # Use 1 as fallback so rows with zero/undefined median are left unchanged
+    med = np.where((med == 0) | ~np.isfinite(med), 1.0, med)
+    if log:
+        return arr - med[:, None]
+    else:
+        return arr / med[:, None]
+
+
+def _safe_row_normalize(v):
+    """Row-standardize; rows with std=0 are mean-subtracted only (result is zeros)."""
+    mu = np.mean(v, axis=1)
+    sigma = np.std(v, axis=1)
+    sigma = np.where(sigma == 0, 1.0, sigma)
+    return (v - mu[:, None]) / sigma[:, None]
+
+
+def apply_runtime_patches():
+    """Swap in all-zero-row-safe normalizers for every caller inside ss_opm.
+
+    Train and predict run the same preprocessing chain, so both have to call this
+    before `PrePostProcessing.preprocess()`.
+    """
+    import ss_opm.utility.nonzero_median_normalize as _mnm_module
+    import ss_opm.utility.row_normalize as _rn_module
+    import ss_opm.pre_post_processing.pre_post_processing as _pp_module
+
+    # patch the source modules so every 'from X import Y' binding stays in sync
+    _mnm_module.median_normalize = _safe_median_normalize
+    _rn_module.row_normalize = _safe_row_normalize
+    # and the names already bound inside pre_post_processing's namespace
+    _pp_module.median_normalize = _safe_median_normalize
+    _pp_module.row_normalize = _safe_row_normalize
diff --git a/src/methods/ss_opm/ss_opm_predict/script.py b/src/methods/ss_opm/ss_opm_predict/script.py
index 1c9e5fe7..ab7b02d3 100644
--- a/src/methods/ss_opm/ss_opm_predict/script.py
+++ b/src/methods/ss_opm/ss_opm_predict/script.py
@@ -27,7 +27,9 @@
 ## VIASH END
 
 sys.path.append(meta['resources_dir'])
-from ss_opm_common import build_metadata, to_sparse_csr
+from ss_opm_common import apply_runtime_patches, build_metadata, to_sparse_csr
+
+apply_runtime_patches()
 
 # ---- Load task info ----
 with open(os.path.join(par['input_model'], 'task_info.pickle'), 'rb') as f:
diff --git a/src/methods/ss_opm/ss_opm_train/script.py b/src/methods/ss_opm/ss_opm_train/script.py
index c714098e..b5f2d16c 100644
--- a/src/methods/ss_opm/ss_opm_train/script.py
+++ b/src/methods/ss_opm/ss_opm_train/script.py
@@ -34,41 +34,9 @@
 ## VIASH END
 
 sys.path.append(meta['resources_dir'])
-from ss_opm_common import build_metadata, to_sparse_csr
-
-# Monkey-patch the source utility modules so ALL callers (pre_post_processing,
-# IterativeSVDImputator, etc.) use safe versions that handle all-zero/all-NaN rows.
-import ss_opm.utility.nonzero_median_normalize as _mnm_module
-import ss_opm.utility.row_normalize as _rn_module
-import ss_opm.pre_post_processing.pre_post_processing as _pp_module
-
-def _safe_median_normalize(values, ignore_zero=True, log=False):
-    """Median-normalize rows, substituting 1 (identity) when the median is 0 or NaN."""
-    arr = np.asarray(values.toarray() if hasattr(values, 'toarray') else values, dtype=float).copy()
-    for_median = arr.copy()
-    if ignore_zero:
-        for_median[for_median == 0] = np.nan
-    med = np.nanquantile(for_median, q=0.5, axis=1)
-    # Use 1 as fallback so rows with zero/undefined median are left unchanged
-    med = np.where((med == 0) | ~np.isfinite(med), 1.0, med)
-    if log:
-        return arr - med[:, None]
-    else:
-        return arr / med[:, None]
-
-def _safe_row_normalize(v):
-    """Row-standardize; rows with std=0 are mean-subtracted only (result is zeros)."""
-    mu = np.mean(v, axis=1)
-    sigma = np.std(v, axis=1)
-    sigma = np.where(sigma == 0, 1.0, sigma)
-    return (v - mu[:, None]) / sigma[:, None]
-
-# Patch the source modules so every 'from X import Y' binding stays in sync
-_mnm_module.median_normalize = _safe_median_normalize
-_rn_module.row_normalize = _safe_row_normalize
-# Also patch the names already bound inside pre_post_processing's namespace
-_pp_module.median_normalize = _safe_median_normalize
-_pp_module.row_normalize = _safe_row_normalize
+from ss_opm_common import apply_runtime_patches, build_metadata, to_sparse_csr
+
+apply_runtime_patches()
 
 # The SVD decomposer components are stored as float64 tensors inside
 # MultiEncoderDecoderModule, but the neural-network outputs are float32.

From 3a6222fb4963c1e009bb3f7d4b5e99aaf23d2650 Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 21:22:24 +0200
Subject: [PATCH 11/14] fall back to a batch split when simple_mlp's site parse
 matches nothing

---
 src/methods/simple_mlp/resources/utils.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/methods/simple_mlp/resources/utils.py b/src/methods/simple_mlp/resources/utils.py
index d001b8e0..f37e54c2 100644
--- a/src/methods/simple_mlp/resources/utils.py
+++ b/src/methods/simple_mlp/resources/utils.py
@@ -1,3 +1,4 @@
+import numpy as np
 import yaml
 from collections import namedtuple
 
@@ -11,8 +12,18 @@ def to_site_donor(data):
 
 
 def split(tr1, tr2, fold):
-    df = to_site_donor(tr1) 
-    mask = df['site'] == f's{fold+1}'
+    df = to_site_donor(tr1)
+    mask = (df['site'] == f's{fold+1}').values
+
+    # batch labels outside the NeurIPS 2021 s{site}d{donor} naming match no site, which
+    # would leave the validation half empty; hold out every third batch instead
+    if not mask.any():
+        batches = sorted(df['batch'].unique())
+        if len(batches) >= 3:
+            mask = df['batch'].isin(batches[fold::3]).values
+        else:
+            mask = np.arange(len(df)) % 3 == fold
+
     maskr = ~mask
 
     Xt = tr1[mask].layers["normalized"].toarray()

From 39b6391ac1f426e2cf1fd657856fcade9b676c58 Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 21:22:24 +0200
Subject: [PATCH 12/14] hold guanlab_dengkw_pm's blas pool to the cores it was
 allotted

---
 src/methods/guanlab_dengkw_pm/config.vsh.yaml | 3 ++-
 src/methods/guanlab_dengkw_pm/script.py       | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/methods/guanlab_dengkw_pm/config.vsh.yaml b/src/methods/guanlab_dengkw_pm/config.vsh.yaml
index fb4b9c20..4f1fc942 100644
--- a/src/methods/guanlab_dengkw_pm/config.vsh.yaml
+++ b/src/methods/guanlab_dengkw_pm/config.vsh.yaml
@@ -39,10 +39,11 @@ engines:
     image: openproblems/base_python:1
     setup:
       - type: python
-        packages: 
+        packages:
           - scikit-learn
           - pandas
           - numpy
+          - threadpoolctl
 runners:
   - type: executable
   - type: nextflow
diff --git a/src/methods/guanlab_dengkw_pm/script.py b/src/methods/guanlab_dengkw_pm/script.py
index 3a0fd029..9e57faa4 100644
--- a/src/methods/guanlab_dengkw_pm/script.py
+++ b/src/methods/guanlab_dengkw_pm/script.py
@@ -5,6 +5,7 @@
 from sklearn.decomposition import TruncatedSVD
 from sklearn.gaussian_process.kernels import RBF
 from sklearn.kernel_ridge import KernelRidge
+from threadpoolctl import threadpool_limits
 
 ## VIASH START
 par = {
@@ -24,6 +25,10 @@
 sys.path.append(meta['resources_dir'])
 from exit_codes import exit_non_applicable
 
+# BLAS otherwise sizes its pool from the node's core count, not the cores allotted to us
+if meta.get('cpus'):
+    threadpool_limits(meta['cpus'])
+
 
 ## Removed PCA and normalization steps, as they arr already performed with the input data
 print('Reading input files', flush=True)

From d905dd96c448a2a958d34fc8906fef9ed994e79d Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 21:22:24 +0200
Subject: [PATCH 13/14] cap cellmapper below 0.3, which has twice renamed the
 obsm key

---
 src/methods/cellmapper_linear/config.vsh.yaml | 2 +-
 src/methods/cellmapper_scvi/config.vsh.yaml   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/methods/cellmapper_linear/config.vsh.yaml b/src/methods/cellmapper_linear/config.vsh.yaml
index b6def679..94cae038 100644
--- a/src/methods/cellmapper_linear/config.vsh.yaml
+++ b/src/methods/cellmapper_linear/config.vsh.yaml
@@ -66,7 +66,7 @@ engines:
     setup:
       - type: python
         packages: 
-          - cellmapper>=0.2.6
+          - cellmapper>=0.2.6,<0.3
 runners:
   - type: executable
   - type: nextflow
diff --git a/src/methods/cellmapper_scvi/config.vsh.yaml b/src/methods/cellmapper_scvi/config.vsh.yaml
index c54b1963..aff93911 100644
--- a/src/methods/cellmapper_scvi/config.vsh.yaml
+++ b/src/methods/cellmapper_scvi/config.vsh.yaml
@@ -73,7 +73,7 @@ engines:
     setup:
       - type: python
         packages: 
-          - cellmapper>=0.2.6
+          - cellmapper>=0.2.6,<0.3
           - scvi-tools>=1.3.0
           - muon>=0.1.6
 

From 56e78b42b86137f1885b777e7427146a19c99a23 Mon Sep 17 00:00:00 2001
From: Robrecht Cannoodt <rcannood@gmail.com>
Date: Fri, 31 Jul 2026 21:22:42 +0200
Subject: [PATCH 14/14] update changelog

---
 CHANGELOG.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 89cc2fe8..4477e4f7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -90,6 +90,14 @@
 
 * `ss_opm`: Compute the per-cell statistics in `build_metadata()` one row block at a time. Densifying the whole normalized layer asked for a 222 GiB allocation on the multiome datasets, which fits on no node (PR #59).
 
+* `ss_opm_predict`: Apply the all-zero-row-safe `median_normalize()` and `row_normalize()` that `ss_opm_train` already monkey-patched in. Predict ran the originals, so an all-zero row produced a `NaN` median and the preprocessing chain handed `NaN` to `TruncatedSVD`. Both now call `apply_runtime_patches()` from `ss_opm_common` (PR #59).
+
+* `simple_mlp_train`: Hold out every third batch when the batch labels do not follow the NeurIPS 2021 `s{site}d{donor}` naming. `split()` matched `site` against `s1`/`s2`/`s3`, so on the 2022 pbmc datasets every fold got an empty validation half, `valid_RMSE` was never logged, and `.predict(ckpt_path="best")` found no checkpoint (PR #59).
+
+* `guanlab_dengkw_pm`: Limit the BLAS thread pool to `meta["cpus"]`. Nothing constrains cores on the cluster, so kernel ridge sized its pool from the node's 64 cores against a 30-core allocation (PR #59).
+
+* `cellmapper_linear`, `cellmapper_scvi`: Require `cellmapper>=0.2.6,<0.3`. The package has twice renamed the `map_obsm` output key under a compatible-looking version bump, each time breaking these components on every dataset (PR #59).
+
 # task_predict_modality 0.1.1
 
 ## NEW FUNCTIONALITY