Fixes

scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh

@@ -8,7 +8,9 @@ cd "$REPO_ROOT"
 
 set -e
 
-publish_dir="s3://openproblems-data/resources/datasets"
+# store the loader output locally, mirroring the process_datasets layout ($id/)
+# under the sibling raw/ folder (same as the other *_nebius.sh spatial scripts)
+publish_dir="/scratch/task_ist_preprocessing/raw"
 
 cat > /tmp/params.yaml << HERE
 param_list:

scripts/run_benchmark/param_sweep/binning_params.yaml

@@ -0,0 +1,39 @@
+# Parameter sweep for the binning segmentation method (the "poor segmentation"
+# baseline: a fixed square grid of pseudo-cells, no image content used).
+#
+# Shared source of truth for both run_test_binning_local.sh (read as a local file)
+# and run_test_binning_nebius.sh (read from GitHub via a raw URL, since the Nebius
+# compute env pulls the repo but cannot see the launch host's local files).
+#
+# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting
+# (src/workflows/run_benchmark/main.nf). For every method the workflow builds:
+#   * one "default" variant using the `default:` args below, and
+#   * one extra variant per value in each `sweep:` list, with that ONE arg overridden.
+# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not
+# a full grid), so total binning variants = 1 default + sum(sweep list lengths) = 6.
+#
+# See src/methods_segmentation/binning/NOTES.md ("Optimization / tuning") for the
+# rationale. The ONLY knob is the bin edge length. We sweep it in MICRONS
+# (--bin_size_um), the physically meaningful unit, rather than raw pixels: the
+# component converts um -> pixels from the image's coordinate transform. On this
+# benchmark's standardized raw_ist grid (~1 um/pixel, target_unit_to_pixels=1) the
+# two coincide, so a value in um is ~ the same value in px.
+#
+# NOTE: --bin_size_um is a NEWLY EXPOSED argument. It needs `viash ns build` +
+# a binning container rebuild before it takes effect (see check-component). Until
+# then, swap `bin_size_um` for the pre-existing `bin_size` (pixels) below.
+parameters:
+  binning:
+    # Baseline == the shipped 30 px default, expressed in microns (~30 um on the
+    # 1 um/px grid): bins ~2-3 cell diameters across -> coarse pseudo-cells.
+    default:
+      bin_size_um: 30.0
+    sweep:
+      # bin_size_um: bin edge length in microns. Grounded in the ~1 um/px grid and
+      # brain/Xenium cell scale (nuclei ~5-8 um, whole cells ~10-15 um):
+      #   10 -> sub-cell / nucleus scale, many bins per cell (over-segmentation)
+      #   15 -> ~one whole cell per bin (best case for a fixed grid)
+      #   20 -> slightly coarse
+      #   (30 = default, omitted here to avoid a duplicate variant)
+      #   40, 50 -> several cells per bin (strong under-segmentation)
+      bin_size_um: [10.0, 15.0, 20.0, 40.0, 50.0]

scripts/run_benchmark/param_sweep/cellpose_params.yaml

@@ -0,0 +1,67 @@
+# Parameter sweep for the cellpose (Cellpose v3, cyto/nuclei CNN) segmentation method.
+# Shared source of truth for both run_test_cellpose_local.sh (read as a local file)
+# and run_test_cellpose_nebius.sh (read from GitHub via a raw URL, since the Nebius
+# compute env pulls the repo but cannot see the launch host's local files).
+#
+# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting
+# (src/workflows/run_benchmark/main.nf). For every method the workflow builds:
+#   * one "default" variant using the `default:` args below, and
+#   * one extra variant per value in each `sweep:` list, with that ONE arg overridden.
+# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not
+# a full grid), so total cellpose variants = 1 default + sum(sweep list lengths) = 20.
+#
+# See src/methods_segmentation/cellpose/NOTES.md ("Optimization / tuning") for the
+# rationale, tiers, and defaults. Unlike cellposev4, this component's shipped defaults
+# already equal Cellpose's own library defaults (quality-oriented) EXCEPT model_type
+# (cyto vs library cyto3), so this sweep explores model choice + object scale +
+# recall/precision dials rather than "walking back" speed-tuned defaults.
+#
+# NOTE: as of txsim@dev the v3 path runs on CPU (the wrapper never passes gpu=), so
+# these 20 variants x the full downstream pipeline are slow; niter: 50 probes the
+# speed lever. Trim the lists if the budget is tight.
+parameters:
+  cellpose:
+    # Baseline == the component's shipped defaults (== Cellpose library defaults,
+    # except model_type). Applied to every variant; the "default variant" is exactly
+    # this point, so none of these values are repeated in the sweep lists below.
+    default:
+      model_type: cyto
+      diameter: 30.0
+      flow_threshold: 0.4
+      cellprob_threshold: 0.0
+      min_size: 15
+      resample: true
+      normalize: true
+      niter: 0
+    sweep:
+      # ---- Tier 1: highest impact on segmentation quality ----
+      # model_type: the most distinctive v3 knob (v4 has no model choice). On a single
+      # grayscale morphology plane the model matters: nuclei for a nuclear stain, cyto2
+      # (Cellpose 2.0) and cyto3 (Cellpose3 generalist) as improved generalists over
+      # the 2021 cyto default. All four are pre-cached in the image (config docker.run).
+      model_type: [nuclei, cyto2, cyto3]
+      # diameter: cellpose rescales so objects land near diam_mean (~30 px cyto).
+      # Xenium morphology (~0.2125 um/px) => nuclei ~38 px, whole cells ~60-70 px.
+      # 0.0 = auto-estimate via the SizeModel (slower). 30.0 = default (omitted).
+      diameter: [0.0, 40.0, 60.0]
+      # cellprob_threshold: recall<->precision dial (per-pixel logit ~ -6..+6, def 0),
+      # no speed cost. Lower -> recover more/dimmer cells; higher -> drop dim detections.
+      cellprob_threshold: [-2.0, -1.0, 1.0, 2.0]
+      # flow_threshold: flow-error QC (def 0.4). 0.2 = stricter shape filter;
+      # 0.6/0.8 = more permissive (keep cells with higher flow error -> more cells).
+      flow_threshold: [0.2, 0.6, 0.8]
+      # ---- Tier 2: quality/speed trade-offs ----
+      # min_size: def 15. 0 keeps small specks (recall on tiny cells); 50 drops debris.
+      min_size: [0, 50]
+      # resample: BOOLEAN, default true (the quality setting). Only non-default value
+      # is false -> dynamics on the downsampled grid (faster, coarser boundaries).
+      resample: [false]
+      # augment: BOOLEAN, default false. Only non-default is true -> test-time
+      # augmentation (accuracy ceiling at ~4-8x cost).
+      augment: [true]
+      # normalize: BOOLEAN, default true. Only non-default is false -> no percentile
+      # normalization (probes intensity sensitivity; usually worse for iST).
+      normalize: [false]
+      # niter: def 0 = auto (~200 at resample). 50 = fewer dynamics iterations ->
+      # faster (meaningful because the v3 path is CPU-bound).
+      niter: [50]

scripts/run_benchmark/param_sweep/run_test_binning_local.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# Test run: all default methods + binning segmentation, with a parameter sweep
+# over the one binning knob (bin edge length, in microns).
+# See src/methods_segmentation/binning/NOTES.md ("Optimization / tuning").
+#
+# NOTE: binning is a light CPU-only method (a numpy grid, no model, no image
+# content used), so it runs comfortably on any Docker host. It is the deliberately
+# "poor" segmentation baseline.
+#
+# NOTE: the sweep uses --bin_size_um, a NEWLY EXPOSED argument. Run
+# 'viash ns build --setup cachedbuild' (or scripts/project/build_all_docker_containers.sh)
+# so the regenerated binning container knows the arg before launching.
+
+# get the root of the directory
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+# ensure that the command below is run from the root of the repository
+cd "$REPO_ROOT"
+
+set -e
+
+echo "Running benchmark on test data (defaults + binning sweep)"
+echo "  Make sure to run 'scripts/project/build_all_docker_containers.sh'!"
+
+# generate a unique id
+RUN_ID="testrun_binning_$(date +%Y-%m-%d_%H-%M-%S)"
+publish_dir="temp/results/${RUN_ID}"
+
+cat > /tmp/params_settings.yaml << HERE
+default_methods:
+  - custom_segmentation
+  - basic_transcript_assignment
+  - basic_count_aggregation
+  - basic_qc_filter
+  - alpha_shapes
+  - normalize_by_volume
+  - tacco
+  - no_correction
+segmentation_methods:
+  - custom_segmentation
+  - binning
+  # - cellpose
+  # - cellposev4
+  # - stardist
+  # - watershed
+transcript_assignment_methods:
+  - basic_transcript_assignment
+count_aggregation_methods:
+  - basic_count_aggregation
+qc_filtering_methods:
+  - basic_qc_filter
+volume_calculation_methods:
+  - alpha_shapes
+normalization_methods:
+  - normalize_by_volume
+celltype_annotation_methods:
+  - ssam
+expression_correction_methods:
+  - no_correction
+gene_efficiency_correction_methods:
+  - no_correction
+method_parameters_yaml: $REPO_ROOT/scripts/run_benchmark/binning_params.yaml
+HERE
+
+# Write the parameters to file (input_states version, NOTE: enable `-entry auto` for this)
+cat > /tmp/params.yaml << HERE
+input_states: resources_test/task_ist_preprocessing/**/state.yaml
+rename_keys: 'input_sc:output_sc;input_sp:output_sp'
+save_spatial_data: false
+settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)'
+output_state: "state.yaml"
+publish_dir: "$publish_dir"
+HERE
+
+# The binning parameter sweep lives in a committed file (single source of truth,
+# shared with run_test_binning_nebius.sh): scripts/run_benchmark/binning_params.yaml
+# It defines a `default:` variant + one-arg-at-a-time `sweep:` variants (a "star"
+# around the default, not a grid) => 6 binning variants. Edit that file to change
+# the sweep. Referenced via $REPO_ROOT above so local Nextflow reads it directly.
+
+nextflow run . \
+  -main-script target/nextflow/workflows/run_benchmark/main.nf \
+  -profile docker \
+  -resume \
+  -entry auto \
+  -c common/nextflow_helpers/labels_ci.config \
+  -params-file /tmp/params.yaml

scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+# Nebius test run: all default methods + binning segmentation, with a parameter
+# sweep over the one binning knob (bin edge length, in microns).
+# See src/methods_segmentation/binning/NOTES.md ("Optimization / tuning").
+# Local sibling: run_test_binning_local.sh
+#
+# binning is a light CPU-only method (a numpy grid, no model), so it runs on the
+# standard (non-GPU) compute env with no `gpu` label.
+#
+# NOTE: the sweep uses --bin_size_um, a NEWLY EXPOSED argument. The image on
+# ghcr must be rebuilt (viash ns build + container rebuild, revision build/main)
+# so the binning container knows the arg before launching (see check-component).
+#
+# PARAMS-FILE CAVEAT (why this differs from the local script):
+#   `tw launch --params-file` is read client-side, but `method_parameters_yaml`
+#   is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow
+#   file()). A local /tmp path does not exist there, and /scratch (where results
+#   publish) is READ-ONLY from the launch host — which is why the binning
+#   method_params block is commented out in run_test_nebius.sh. file() does stage
+#   http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED
+#   file (scripts/run_benchmark/binning_params.yaml) and read it from GitHub via
+#   its raw URL. => the params file must be committed AND PUSHED to $params_branch
+#   before launching (edit the file there, not here, to change the sweep).
+
+# get the root of the directory
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+# ensure that the command below is run from the root of the repository
+cd "$REPO_ROOT"
+
+set -e
+
+resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing
+# Results publish to /scratch — created and written by the cloud compute env, so
+# the launcher does NOT create it here (it is read-only from the launch host).
+publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_binning"
+
+# The sweep lives in a committed file, read from GitHub at runtime. $params_branch
+# defaults to the branch you are on; the file must be pushed there on GitHub. (This
+# is independent of --revision below, which selects the pipeline CODE to run.)
+params_repo="openproblems-bio/task_ist_preprocessing"
+params_branch="$(git rev-parse --abbrev-ref HEAD)"
+params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/binning_params.yaml"
+
+cat > /tmp/params_settings.yaml << HERE
+default_methods:
+  - custom_segmentation
+  - basic_transcript_assignment
+  - basic_count_aggregation
+  - basic_qc_filter
+  - alpha_shapes
+  - normalize_by_volume
+  - tacco
+  - no_correction
+segmentation_methods:
+  - custom_segmentation
+  - binning
+#  - cellpose
+#  - cellposev4
+#  - stardist
+#  - watershed
+transcript_assignment_methods:
+  - basic_transcript_assignment
+count_aggregation_methods:
+  - basic_count_aggregation
+qc_filtering_methods:
+  - basic_qc_filter
+volume_calculation_methods:
+  - alpha_shapes
+normalization_methods:
+  - normalize_by_volume
+celltype_annotation_methods:
+  - tacco
+expression_correction_methods:
+  - no_correction
+gene_efficiency_correction_methods:
+  - no_correction
+method_parameters_yaml: $params_url
+HERE
+
+# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this)
+cat > /tmp/params.yaml << HERE
+input_states: $resources_test_s3/**/state.yaml
+rename_keys: 'input_sc:output_sc;input_sp:output_sp'
+save_spatial_data: false
+settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)'
+output_state: "state.yaml"
+publish_dir: "$publish_dir"
+HERE
+
+# Fail early with a clear message if the params file isn't reachable on GitHub yet.
+if ! curl -fsSL -o /dev/null "$params_url"; then
+  echo "ERROR: params file not reachable at:" >&2
+  echo "  $params_url" >&2
+  echo "Commit and push scripts/run_benchmark/binning_params.yaml to '$params_branch' first." >&2
+  exit 1
+fi
+
+tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
+  --revision build/main \
+  --pull-latest \
+  --main-script target/nextflow/workflows/run_benchmark/main.nf \
+  --workspace 167877437119966 \
+  --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \
+  --params-file /tmp/params.yaml \
+  --entry-name auto \
+  --config src/base/labels_nebius.config \
+  --labels task_ist_preprocessing,test,binning