diff --git a/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh b/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh index c12a28b1b..dd3232ef4 100644 --- a/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh +++ b/scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh @@ -8,7 +8,9 @@ cd "$REPO_ROOT" set -e -publish_dir="s3://openproblems-data/resources/datasets" +# store the loader output locally, mirroring the process_datasets layout ($id/) +# under the sibling raw/ folder (same as the other *_nebius.sh spatial scripts) +publish_dir="/scratch/task_ist_preprocessing/raw" cat > /tmp/params.yaml << HERE param_list: diff --git a/scripts/run_benchmark/param_sweep/binning_params.yaml b/scripts/run_benchmark/param_sweep/binning_params.yaml new file mode 100644 index 000000000..7ca9ec45e --- /dev/null +++ b/scripts/run_benchmark/param_sweep/binning_params.yaml @@ -0,0 +1,39 @@ +# Parameter sweep for the binning segmentation method (the "poor segmentation" +# baseline: a fixed square grid of pseudo-cells, no image content used). +# +# Shared source of truth for both run_test_binning_local.sh (read as a local file) +# and run_test_binning_nebius.sh (read from GitHub via a raw URL, since the Nebius +# compute env pulls the repo but cannot see the launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total binning variants = 1 default + sum(sweep list lengths) = 6. +# +# See src/methods_segmentation/binning/NOTES.md ("Optimization / tuning") for the +# rationale. The ONLY knob is the bin edge length. We sweep it in MICRONS +# (--bin_size_um), the physically meaningful unit, rather than raw pixels: the +# component converts um -> pixels from the image's coordinate transform. On this +# benchmark's standardized raw_ist grid (~1 um/pixel, target_unit_to_pixels=1) the +# two coincide, so a value in um is ~ the same value in px. +# +# NOTE: --bin_size_um is a NEWLY EXPOSED argument. It needs `viash ns build` + +# a binning container rebuild before it takes effect (see check-component). Until +# then, swap `bin_size_um` for the pre-existing `bin_size` (pixels) below. +parameters: + binning: + # Baseline == the shipped 30 px default, expressed in microns (~30 um on the + # 1 um/px grid): bins ~2-3 cell diameters across -> coarse pseudo-cells. + default: + bin_size_um: 30.0 + sweep: + # bin_size_um: bin edge length in microns. Grounded in the ~1 um/px grid and + # brain/Xenium cell scale (nuclei ~5-8 um, whole cells ~10-15 um): + # 10 -> sub-cell / nucleus scale, many bins per cell (over-segmentation) + # 15 -> ~one whole cell per bin (best case for a fixed grid) + # 20 -> slightly coarse + # (30 = default, omitted here to avoid a duplicate variant) + # 40, 50 -> several cells per bin (strong under-segmentation) + bin_size_um: [10.0, 15.0, 20.0, 40.0, 50.0] diff --git a/scripts/run_benchmark/param_sweep/cellpose_params.yaml b/scripts/run_benchmark/param_sweep/cellpose_params.yaml new file mode 100644 index 000000000..3887fb1e7 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/cellpose_params.yaml @@ -0,0 +1,67 @@ +# Parameter sweep for the cellpose (Cellpose v3, cyto/nuclei CNN) segmentation method. +# Shared source of truth for both run_test_cellpose_local.sh (read as a local file) +# and run_test_cellpose_nebius.sh (read from GitHub via a raw URL, since the Nebius +# compute env pulls the repo but cannot see the launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total cellpose variants = 1 default + sum(sweep list lengths) = 20. +# +# See src/methods_segmentation/cellpose/NOTES.md ("Optimization / tuning") for the +# rationale, tiers, and defaults. Unlike cellposev4, this component's shipped defaults +# already equal Cellpose's own library defaults (quality-oriented) EXCEPT model_type +# (cyto vs library cyto3), so this sweep explores model choice + object scale + +# recall/precision dials rather than "walking back" speed-tuned defaults. +# +# NOTE: as of txsim@dev the v3 path runs on CPU (the wrapper never passes gpu=), so +# these 20 variants x the full downstream pipeline are slow; niter: 50 probes the +# speed lever. Trim the lists if the budget is tight. +parameters: + cellpose: + # Baseline == the component's shipped defaults (== Cellpose library defaults, + # except model_type). Applied to every variant; the "default variant" is exactly + # this point, so none of these values are repeated in the sweep lists below. + default: + model_type: cyto + diameter: 30.0 + flow_threshold: 0.4 + cellprob_threshold: 0.0 + min_size: 15 + resample: true + normalize: true + niter: 0 + sweep: + # ---- Tier 1: highest impact on segmentation quality ---- + # model_type: the most distinctive v3 knob (v4 has no model choice). On a single + # grayscale morphology plane the model matters: nuclei for a nuclear stain, cyto2 + # (Cellpose 2.0) and cyto3 (Cellpose3 generalist) as improved generalists over + # the 2021 cyto default. All four are pre-cached in the image (config docker.run). + model_type: [nuclei, cyto2, cyto3] + # diameter: cellpose rescales so objects land near diam_mean (~30 px cyto). + # Xenium morphology (~0.2125 um/px) => nuclei ~38 px, whole cells ~60-70 px. + # 0.0 = auto-estimate via the SizeModel (slower). 30.0 = default (omitted). + diameter: [0.0, 40.0, 60.0] + # cellprob_threshold: recall<->precision dial (per-pixel logit ~ -6..+6, def 0), + # no speed cost. Lower -> recover more/dimmer cells; higher -> drop dim detections. + cellprob_threshold: [-2.0, -1.0, 1.0, 2.0] + # flow_threshold: flow-error QC (def 0.4). 0.2 = stricter shape filter; + # 0.6/0.8 = more permissive (keep cells with higher flow error -> more cells). + flow_threshold: [0.2, 0.6, 0.8] + # ---- Tier 2: quality/speed trade-offs ---- + # min_size: def 15. 0 keeps small specks (recall on tiny cells); 50 drops debris. + min_size: [0, 50] + # resample: BOOLEAN, default true (the quality setting). Only non-default value + # is false -> dynamics on the downsampled grid (faster, coarser boundaries). + resample: [false] + # augment: BOOLEAN, default false. Only non-default is true -> test-time + # augmentation (accuracy ceiling at ~4-8x cost). + augment: [true] + # normalize: BOOLEAN, default true. Only non-default is false -> no percentile + # normalization (probes intensity sensitivity; usually worse for iST). + normalize: [false] + # niter: def 0 = auto (~200 at resample). 50 = fewer dynamics iterations -> + # faster (meaningful because the v3 path is CPU-bound). + niter: [50] diff --git a/scripts/run_benchmark/cellposev4_params.yaml b/scripts/run_benchmark/param_sweep/cellposev4_params.yaml similarity index 100% rename from scripts/run_benchmark/cellposev4_params.yaml rename to scripts/run_benchmark/param_sweep/cellposev4_params.yaml diff --git a/scripts/run_benchmark/param_sweep/run_test_binning_local.sh b/scripts/run_benchmark/param_sweep/run_test_binning_local.sh new file mode 100644 index 000000000..3afe8980e --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_binning_local.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# Test run: all default methods + binning segmentation, with a parameter sweep +# over the one binning knob (bin edge length, in microns). +# See src/methods_segmentation/binning/NOTES.md ("Optimization / tuning"). +# +# NOTE: binning is a light CPU-only method (a numpy grid, no model, no image +# content used), so it runs comfortably on any Docker host. It is the deliberately +# "poor" segmentation baseline. +# +# NOTE: the sweep uses --bin_size_um, a NEWLY EXPOSED argument. Run +# 'viash ns build --setup cachedbuild' (or scripts/project/build_all_docker_containers.sh) +# so the regenerated binning container knows the arg before launching. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +echo "Running benchmark on test data (defaults + binning sweep)" +echo " Make sure to run 'scripts/project/build_all_docker_containers.sh'!" + +# generate a unique id +RUN_ID="testrun_binning_$(date +%Y-%m-%d_%H-%M-%S)" +publish_dir="temp/results/${RUN_ID}" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - binning + # - cellpose + # - cellposev4 + # - stardist + # - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $REPO_ROOT/scripts/run_benchmark/binning_params.yaml +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry auto` for this) +cat > /tmp/params.yaml << HERE +input_states: resources_test/task_ist_preprocessing/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# The binning parameter sweep lives in a committed file (single source of truth, +# shared with run_test_binning_nebius.sh): scripts/run_benchmark/binning_params.yaml +# It defines a `default:` variant + one-arg-at-a-time `sweep:` variants (a "star" +# around the default, not a grid) => 6 binning variants. Edit that file to change +# the sweep. Referenced via $REPO_ROOT above so local Nextflow reads it directly. + +nextflow run . \ + -main-script target/nextflow/workflows/run_benchmark/main.nf \ + -profile docker \ + -resume \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + -params-file /tmp/params.yaml diff --git a/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh new file mode 100644 index 000000000..527e0d3d6 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_binning_nebius.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# Nebius test run: all default methods + binning segmentation, with a parameter +# sweep over the one binning knob (bin edge length, in microns). +# See src/methods_segmentation/binning/NOTES.md ("Optimization / tuning"). +# Local sibling: run_test_binning_local.sh +# +# binning is a light CPU-only method (a numpy grid, no model), so it runs on the +# standard (non-GPU) compute env with no `gpu` label. +# +# NOTE: the sweep uses --bin_size_um, a NEWLY EXPOSED argument. The image on +# ghcr must be rebuilt (viash ns build + container rebuild, revision build/main) +# so the binning container knows the arg before launching (see check-component). +# +# PARAMS-FILE CAVEAT (why this differs from the local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/binning_params.yaml) and read it from GitHub via +# its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_binning" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/binning_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - binning +# - cellpose +# - cellposev4 +# - stardist +# - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/binning_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,binning diff --git a/scripts/run_benchmark/param_sweep/run_test_cellpose_local.sh b/scripts/run_benchmark/param_sweep/run_test_cellpose_local.sh new file mode 100644 index 000000000..19ed5de22 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_cellpose_local.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# Test run: all default methods + Cellpose v3 (cellpose, cyto/nuclei CNN) +# segmentation, with a parameter sweep over the optimization levers identified +# for Cellpose v3. See src/methods_segmentation/cellpose/NOTES.md +# ("Optimization / tuning"). +# +# NOTE: despite the component's gpuhighmem label, the v3 path runs on CPU (the +# txsim wrapper never passes gpu= -> models.Cellpose defaults to gpu=False). So +# these 20 variants x the full downstream pipeline are CPU-bound and slow; the +# sweep includes niter: 50 as a speed probe. See the GPU gotcha in NOTES.md. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +echo "Running benchmark on test data (defaults + cellpose v3 sweep)" +echo " Make sure to run 'scripts/project/build_all_docker_containers.sh'!" + +# generate a unique id +RUN_ID="testrun_cellpose_$(date +%Y-%m-%d_%H-%M-%S)" +publish_dir="temp/results/${RUN_ID}" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - cellpose + # - cellposev4 + # - binning + # - stardist + # - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $REPO_ROOT/scripts/run_benchmark/cellpose_params.yaml +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry auto` for this) +cat > /tmp/params.yaml << HERE +input_states: resources_test/task_ist_preprocessing/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# The cellpose parameter sweep lives in a committed file (single source of truth, +# shared with run_test_cellpose_nebius.sh): scripts/run_benchmark/cellpose_params.yaml +# It defines a `default:` variant + one-arg-at-a-time `sweep:` variants (a "star" +# around the default, not a grid) => 20 cellpose variants. Edit that file to +# change the sweep. Referenced via $REPO_ROOT above so local Nextflow reads it directly. + +nextflow run . \ + -main-script target/nextflow/workflows/run_benchmark/main.nf \ + -profile docker \ + -resume \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + -params-file /tmp/params.yaml diff --git a/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh new file mode 100644 index 000000000..e3458e9a6 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_cellpose_nebius.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# Nebius test run: all default methods + Cellpose v3 (cellpose, cyto/nuclei CNN) +# segmentation, with a parameter sweep over the optimization levers identified +# for Cellpose v3. See src/methods_segmentation/cellpose/NOTES.md +# ("Optimization / tuning"). Local sibling: run_test_cellpose_local.sh +# +# NOTE: the component carries a gpuhighmem label (so it is scheduled onto a GPU +# node), but as of txsim@dev the v3 code path runs on CPU (the wrapper never +# passes gpu=). The `gpu` run label below matches the config's scheduling; it +# does NOT make the model GPU-accelerated. See the GPU gotcha in NOTES.md. +# +# PARAMS-FILE CAVEAT (why this differs from the local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/cellpose_params.yaml) and read it from GitHub via +# its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_cellpose" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/cellpose_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - cellpose +# - cellposev4 +# - binning +# - stardist +# - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/cellpose_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,cellpose,gpu diff --git a/scripts/run_benchmark/param_sweep/run_test_cellposev4_local.sh b/scripts/run_benchmark/param_sweep/run_test_cellposev4_local.sh new file mode 100644 index 000000000..9f2f923f0 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_cellposev4_local.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# Test run: all default methods + Cellpose 4 (cellposev4) segmentation, with a +# parameter sweep over the optimization levers identified for Cellpose-SAM. +# See src/methods_segmentation/cellposev4/NOTES.md ("Optimization / tuning"). +# +# NOTE: cellposev4 (cpsam / SAM ViT) is GPU-heavy. Run on a CUDA-capable Docker +# host; on CPU it is very slow (the documented reason it is off the standard CI +# test path). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +echo "Running benchmark on test data (defaults + cellposev4 sweep)" +echo " Make sure to run 'scripts/project/build_all_docker_containers.sh'!" + +# generate a unique id +RUN_ID="testrun_cellposev4_$(date +%Y-%m-%d_%H-%M-%S)" +publish_dir="temp/results/${RUN_ID}" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - cellposev4 + # - cellpose + # - binning + # - stardist + # - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $REPO_ROOT/scripts/run_benchmark/cellposev4_params.yaml +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry auto` for this) +cat > /tmp/params.yaml << HERE +input_states: resources_test/task_ist_preprocessing/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# The cellposev4 parameter sweep lives in a committed file (single source of truth, +# shared with run_test_cellposev4_nebius.sh): scripts/run_benchmark/cellposev4_params.yaml +# It defines a `default:` variant + one-arg-at-a-time `sweep:` variants (a "star" +# around the default, not a grid) => 19 cellposev4 variants. Edit that file to +# change the sweep. Referenced via $REPO_ROOT above so local Nextflow reads it directly. + +nextflow run . \ + -main-script target/nextflow/workflows/run_benchmark/main.nf \ + -profile docker \ + -resume \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + -params-file /tmp/params.yaml diff --git a/scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh new file mode 100644 index 000000000..5c092195e --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_cellposev4_nebius.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Nebius test run: all default methods + Cellpose 4 (cellposev4) segmentation, +# with a parameter sweep over the optimization levers identified for Cellpose-SAM. +# See src/methods_segmentation/cellposev4/NOTES.md ("Optimization / tuning"). +# Local sibling: run_test_cellposev4_local.sh +# +# cellposev4 (cpsam / SAM ViT) is GPU-heavy, hence the `gpu` label and the Nebius +# GPU compute env. +# +# PARAMS-FILE CAVEAT (why this differs from the local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/cellposev4_params.yaml) and read it from GitHub via +# its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_cellposev4" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/cellposev4_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - cellposev4 +# - cellpose +# - binning +# - stardist +# - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/cellposev4_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,cellposev4,gpu diff --git a/scripts/run_benchmark/param_sweep/run_test_stardist_local.sh b/scripts/run_benchmark/param_sweep/run_test_stardist_local.sh new file mode 100644 index 000000000..e6b168151 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_stardist_local.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Test run: all default methods + StarDist2D (stardist) segmentation, with a +# parameter sweep over the optimization levers identified for StarDist. +# See src/methods_segmentation/stardist/NOTES.md ("Optimization / tuning"). +# +# NOTE: stardist runs on TensorFlow via a GPU-capable base image +# (openproblems/base_tensorflow_nvidia). It falls back to CPU if no GPU is present +# (slower, but far lighter than cellposev4's SAM ViT). Run on a CUDA-capable Docker +# host for the full sweep. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +echo "Running benchmark on test data (defaults + stardist sweep)" +echo " Make sure to run 'scripts/project/build_all_docker_containers.sh'!" + +# generate a unique id +RUN_ID="testrun_stardist_$(date +%Y-%m-%d_%H-%M-%S)" +publish_dir="temp/results/${RUN_ID}" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - stardist + # - cellpose + # - cellposev4 + # - binning + # - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $REPO_ROOT/scripts/run_benchmark/stardist_params.yaml +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry auto` for this) +cat > /tmp/params.yaml << HERE +input_states: resources_test/task_ist_preprocessing/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# The stardist parameter sweep lives in a committed file (single source of truth, +# shared with run_test_stardist_nebius.sh): scripts/run_benchmark/stardist_params.yaml +# It defines a `default:` variant + one-arg-at-a-time `sweep:` variants (a "star" +# around the default, not a grid) => 13 stardist variants. Edit that file to change +# the sweep. Referenced via $REPO_ROOT above so local Nextflow reads it directly. + +nextflow run . \ + -main-script target/nextflow/workflows/run_benchmark/main.nf \ + -profile docker \ + -resume \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + -params-file /tmp/params.yaml diff --git a/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh new file mode 100644 index 000000000..21d22c6bc --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_stardist_nebius.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Nebius test run: all default methods + StarDist2D (stardist) segmentation, with a +# parameter sweep over the optimization levers identified for StarDist. +# See src/methods_segmentation/stardist/NOTES.md ("Optimization / tuning"). +# Local sibling: run_test_stardist_local.sh +# +# stardist runs on TensorFlow and is scheduled with the `gpu` label (its config uses +# a GPU-capable base image), hence the Nebius GPU compute env. +# +# PARAMS-FILE CAVEAT (why this differs from the local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/stardist_params.yaml) and read it from GitHub via +# its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_stardist" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/stardist_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - stardist +# - cellpose +# - cellposev4 +# - binning +# - watershed +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/stardist_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,stardist,gpu diff --git a/scripts/run_benchmark/param_sweep/run_test_watershed_local.sh b/scripts/run_benchmark/param_sweep/run_test_watershed_local.sh new file mode 100644 index 000000000..7ea207252 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_watershed_local.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Test run: all default methods + watershed segmentation, with a parameter sweep +# over the Tier-1 optimization levers identified for classic marker-controlled +# watershed. See src/methods_segmentation/watershed/NOTES.md ("Optimization / +# tuning"). +# +# watershed is a CPU method (no GPU needed): the txsim classic pipeline +# normalize -> contrast -> blur -> threshold -> post-process -> distance transform +# -> local-maxima markers -> watershed -> background filter. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +echo "Running benchmark on test data (defaults + watershed sweep)" +echo " Make sure to run 'scripts/project/build_all_docker_containers.sh'!" + +# generate a unique id +RUN_ID="testrun_watershed_$(date +%Y-%m-%d_%H-%M-%S)" +publish_dir="temp/results/${RUN_ID}" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - watershed + # - cellpose + # - cellposev4 + # - binning + # - stardist +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $REPO_ROOT/scripts/run_benchmark/watershed_params.yaml +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry auto` for this) +cat > /tmp/params.yaml << HERE +input_states: resources_test/task_ist_preprocessing/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# The watershed parameter sweep lives in a committed file (single source of truth, +# shared with run_test_watershed_nebius.sh): scripts/run_benchmark/watershed_params.yaml +# It defines a `default:` variant + one-arg-at-a-time `sweep:` variants (a "star" +# around the default, not a grid) => 12 watershed variants. Edit that file to +# change the sweep. Referenced via $REPO_ROOT above so local Nextflow reads it directly. + +nextflow run . \ + -main-script target/nextflow/workflows/run_benchmark/main.nf \ + -profile docker \ + -resume \ + -entry auto \ + -c common/nextflow_helpers/labels_ci.config \ + -params-file /tmp/params.yaml diff --git a/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh new file mode 100644 index 000000000..657635223 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_watershed_nebius.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Nebius test run: all default methods + watershed segmentation, with a parameter +# sweep over the Tier-1 optimization levers identified for classic marker-controlled +# watershed. See src/methods_segmentation/watershed/NOTES.md ("Optimization / tuning"). +# Local sibling: run_test_watershed_local.sh +# +# watershed is a CPU method (midcpu/midmem/hightime) — NO gpu label, and it reuses +# the standard (non-GPU) Nebius test compute env. +# +# PARAMS-FILE CAVEAT (why this differs from the local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/watershed_params.yaml) and read it from GitHub via +# its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_watershed" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/watershed_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation + - watershed +# - cellpose +# - cellposev4 +# - binning +# - stardist +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/watershed_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,watershed diff --git a/scripts/run_benchmark/param_sweep/stardist_params.yaml b/scripts/run_benchmark/param_sweep/stardist_params.yaml new file mode 100644 index 000000000..434c251a1 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/stardist_params.yaml @@ -0,0 +1,43 @@ +# Parameter sweep for the stardist (StarDist2D) segmentation method. +# Shared source of truth for both run_test_stardist_local.sh (read as a local file) +# and run_test_stardist_nebius.sh (read from GitHub via a raw URL, since the Nebius +# compute env pulls the repo but cannot see the launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total stardist variants = 1 default + sum(sweep list lengths) = 13. +# +# See src/methods_segmentation/stardist/NOTES.md ("Optimization / tuning") for the +# rationale, tiers, and defaults. prob_thresh / nms_thresh / scale are the newly +# exposed StarDist knobs (need `viash ns build` + a container rebuild to take effect). +parameters: + stardist: + # Baseline. Only `model` is set explicitly; prob_thresh / nms_thresh / scale are + # intentionally LEFT UNSET so the default variant uses the model's own optimized + # thresholds (0.479071 / 0.3 for 2D_versatile_fluo) and no rescaling — i.e. exactly + # today's production behaviour. Setting them to a fixed number would be wrong for a + # different --model, whose optimized thresholds differ. + default: + model: "2D_versatile_fluo" + sweep: + # ---- Tier 1: highest impact on segmentation quality ---- + # prob_thresh: object-probability threshold (default ~0.479 for 2D_versatile_fluo). + # Pure recall<->precision dial. LOWER -> recover more / dimmer nuclei; HIGHER -> + # keep only confident detections. Span both sides of the default. + prob_thresh: [0.3, 0.4, 0.6, 0.7] + # scale: isotropic rescale before prediction (default = none / 1.0). StarDist's + # analogue of Cellpose's diameter — matches nucleus size to the model's training + # size. Xenium morphology (~0.2125 um/px) => ~8 um nucleus ~= 38 px; sweep down + # (downscale large nuclei) and up (upscale small nuclei). + scale: [0.5, 0.75, 1.5, 2.0] + # nms_thresh: non-max-suppression IoU for overlapping polygons (default 0.3). + # LOWER -> more aggressive suppression (merge/drop touching nuclei); HIGHER -> + # keep more overlapping detections in crowded tissue. + nms_thresh: [0.2, 0.4, 0.5] + # ---- Tier 1: model choice (already exposed) ---- + # 2D_paper_dsb2018 is the other fluorescence/nuclear pretrained model. 2D_versatile_he + # is H&E RGB (wrong for single-channel fluorescence) so it is NOT swept. + model: ["2D_paper_dsb2018"] diff --git a/scripts/run_benchmark/param_sweep/watershed_params.yaml b/scripts/run_benchmark/param_sweep/watershed_params.yaml new file mode 100644 index 000000000..65ac30343 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/watershed_params.yaml @@ -0,0 +1,36 @@ +# Parameter sweep for the `watershed` segmentation method. +# Single source of truth for BOTH run_test_watershed_local.sh (read directly via +# $REPO_ROOT) and run_test_watershed_nebius.sh (read from GitHub via raw URL, so it +# must be committed AND pushed before a Nebius launch). +# +# Expansion is a STAR around `default:`, NOT a grid: one extra variant per sweep +# value, that one arg overridden, everything else = default. +# total variants = 1 (default) + Σ len(sweep list) = 1 + 2 + 3 + 3 + 3 = 12 +# +# The four swept knobs are the Tier-1 levers for classic watershed (see +# src/methods_segmentation/watershed/NOTES.md "Optimization / tuning"). Ranges are +# grounded in Xenium morphology (~0.2125 µm/px => an ~8 µm nucleus ≈ 37 px diameter, +# radius ≈ 18 px, area ≈ 1000 px²). The other ~40 classic-pipeline args stay at +# their component defaults and are intentionally NOT swept. +parameters: + watershed: + default: + threshold_func: local_otsu # adaptive local Otsu (rank.otsu, footprint 50 px) + blur_sigma: 1 # gaussian pre-smoothing sigma (px) + local_maxima_min_distance: 5 # min px between watershed seed markers + post_processing_min_size_1: 64 # remove_small_objects: min mask area (px) + sweep: + # foreground/background mask decision. local_otsu (default) is adaptive; the + # two alternatives use a single global threshold (triangle suits a dominant + # background peak, typical of fluorescence). + threshold_func: [otsu, triangle] + # heavier pre-smoothing suppresses spurious local maxima (fewer over-split + # cells) at the cost of blurring small/dim nuclei. 2-4 px stays sub-nucleus. + blur_sigma: [2, 3, 4] + # merge<->split dial: markers must be >= this many px apart. default 5 is far + # below the nucleus radius (~18 px) and over-splits single nuclei; walking up + # toward the radius merges fragments back into one cell. + local_maxima_min_distance: [10, 15, 20] + # precision dial: drop small foreground blobs before watershed. Stays well + # below a full nucleus area (~1000 px²) so real nuclei survive. + post_processing_min_size_1: [128, 256, 512] diff --git a/src/datasets/loaders/bruker_cosmx/script.py b/src/datasets/loaders/bruker_cosmx/script.py index 902ccdc4b..9317fbe8a 100644 --- a/src/datasets/loaders/bruker_cosmx/script.py +++ b/src/datasets/loaders/bruker_cosmx/script.py @@ -44,7 +44,7 @@ │ └── -polygons.csv ├── (AnalysisResults) └── (RunSummary) ---> the CellLabels folder is not present, but the CellLabels_FXXX.tif files are in the FOV folders. +--> the CellLabels folder is not present, but the CellLabels_FXXX.tif files are in the FOV folders. They are moved to a newly created CellLabels folder. ### Version 2 (Example: CosMx Mouse brain) ### @@ -65,21 +65,30 @@ ├── _tx_file.csv └── -polygons.csv --> the flat files are in a separate zip, they need to be moved to CellStatsDir ( = `DATA_DIR`) ---> as in version 1, the CellLabels folder is not present, but the CellLabels_FXXX.tif files are in the FOV folders. - +--> as in version 1, the CellLabels folder is not present, but the CellLabels_FXXX.tif files are in the FOV folders. ### Version 3 (Example: CosMx lung cancer) ### -this has subdirectories for each sample and an extra sub directories for morphology images. Also, +this has subdirectories for each sample and an extra sub directories for morphology images. Also, the images are given for each z plane. This dataset is covered with the bruker_cosmx_nsclc dataloader. - - - +### Version 4 (Example: CosMx *Normal Liver*, "Raw Data Files" release) ### +NanoString/Bruker never published the AtoMx "flat files" for the public liver FFPE dataset — only the raw +`NormalLiverFiles.zip` (images + segmentation) plus TileDB/Seurat exports. So the raw zip has NO +`_tx_file.csv` / `_fov_positions_file.csv` etc. — `sopa.io.cosmx` would fail at `_infer_dataset_id`. +The raw material to rebuild the flat files IS present, though: + - `AnalysisResults/**/FOV###__complete_code_cell_target_call_coord.csv` — per-transcript rows with local + pixel coords (`x`,`y`), `z`, `target` (gene), `CellId` (per-FOV cell, 0 = unassigned), `CellComp`. + - `RunSummary/latest.fovs.csv` — per-FOV stage position (columns: ..., x_mm, y_mm, ..., fov). + - `CellStatsDir/FOV###/*Cell_Stats_F###.csv` — per-cell `CellId, Area, CenterX, CenterY` (local px). + - `CellStatsDir/FOV###/CellLabels_F###.tif` + `CellStatsDir/Morphology2D/*_F###.TIF` — labels + image. +When the flat files are missing we RECONSTRUCT them (`reconstruct_flat_files`, see below) and then hand the +result to the *same* `sopa.io.cosmx` call, so all the stitching/coordinate math stays sopa's job. """ import os +import re import shutil import zipfile from pathlib import Path @@ -92,6 +101,9 @@ "input_raw": "temp/datasets/bruker_cosmx/HalfBrain.zip", # downloaded from https://smi-public.objects.liquidweb.services/Half%20%20Brain%20simple%20%20files%20.zip "input_flat_files": "temp/datasets/bruker_cosmx/Half Brain simple files .zip", + # For the liver "Raw Data Files" release (no flat files), point input_raw at + # https://smi-public.objects.liquidweb.services/NormalLiverFiles.zip and leave input_flat_files unset; + # the flat files are then reconstructed automatically (see Version 4 above). "segmentation_id": ["cell"], "output": "output.zarr", "dataset_id": "bruker_cosmx/bruker_mouse_brain_cosmx/rep1", @@ -126,12 +138,37 @@ def log(msg): # whole thing next to the staged zip overruns the node's ~220 GB scratch and OOMs (137). SKIP_DIRS = {"Morphology3D", "AnalysisResults", "RunSummary"} +# Prefix used for the flat files we reconstruct ourselves (Version 4). Passed to +# sopa as an explicit dataset_id so it doesn't try to infer one from a filename. +RECON_ID = "cosmx" + def _member_wanted(name: str) -> bool: parts = name.split("/") if name.startswith("__MACOSX/") or parts[-1] == ".DS_Store": return False return not any(part in SKIP_DIRS for part in parts) +# When reconstructing (Version 4), the AnalysisResults/RunSummary that the denylist above +# would drop are exactly what we need. Extract an *allowlist* instead: only the per-FOV +# transcript-decoding CSVs, the FOV positions, the cell stats, the cell labels and the 2D +# morphology — everything else (Morphology3D, Morphology2D_Normalized, RnD, the redundant +# target_call_coord/total_unambiguous CSVs, the imaging metrics) is dropped. Footprint for +# the liver hemisphere: ~61 GB vs ~440 GB of Morphology3D alone. +_RECON_KEEP = [ + re.compile(r"(^|/)CellLabels_F\d+\.tif$", re.I), # per-FOV cell labels + re.compile(r"(^|/)CellLabels/[^/]+\.tif$", re.I), # ... or an already-gathered CellLabels dir + re.compile(r"Cell_Stats_F\d+.*\.csv$", re.I), # per-cell centroids/area + re.compile(r"/Morphology2D/[^/]+\.[Tt][Ii][Ff]$"), # 2D morphology image (NOT Morphology2D_Normalized) + re.compile(r"complete_code_cell_target_call_coord\.csv$"), # per-transcript calls (+ cell assignment) + re.compile(r"\.fovs\.csv$"), # FOV stage positions +] + +def _member_wanted_reconstruct(name: str) -> bool: + parts = name.split("/") + if name.startswith("__MACOSX/") or parts[-1] == ".DS_Store": + return False + return any(pat.search(name) for pat in _RECON_KEEP) + def _open_zip_source(src): """Seekable binary handle for a local path or an s3:// URL (streamed, never fully downloaded).""" s = str(src) @@ -142,18 +179,27 @@ def _open_zip_source(src): return fs.open(s, "rb") return open(s, "rb") -def extract_zip(input_zip, output_dir: Path, strip_root: bool = False): +def _zip_has_flat_files(src) -> bool: + """Peek the zip's central directory (cheap, streamed) for a `*_tx_file.csv` flat file.""" + with _open_zip_source(src) as fh, zipfile.ZipFile(fh, "r") as zip_ref: + return any( + n.endswith("_tx_file.csv") or n.endswith("_tx_file.csv.gz") + for n in zip_ref.namelist() + ) + +def extract_zip(input_zip, output_dir: Path, strip_root: bool = False, wanted=_member_wanted): """ - Stream a zip (local path or s3:// URL) into output_dir, skipping the CosMx - directories sopa never reads (see SKIP_DIRS). Only the wanted members are - transferred, so a remote zip streams just the bytes we keep — it is never - downloaded in full first. + Stream a zip (local path or s3:// URL) into output_dir, keeping only the members for + which `wanted(name)` is True (see `_member_wanted` / `_member_wanted_reconstruct`). Only + the wanted members are transferred, so a remote zip streams just the bytes we keep — it is + never downloaded in full first. Arguments: - input_zip: local path or s3:// URL of the input zip. - output_dir: directory where the (filtered) contents are written. - strip_root: if True and all entries share exactly one top-level directory, strip it. + - wanted: predicate on the member name deciding whether to extract it. """ output_dir = Path(output_dir) kept = skipped = 0 @@ -166,7 +212,7 @@ def extract_zip(input_zip, output_dir: Path, strip_root: bool = False): do_strip = strip_root and len(roots) == 1 for member in members: - if not _member_wanted(member.filename): + if not wanted(member.filename): skipped += 1 continue parts = Path(member.filename).parts @@ -183,13 +229,30 @@ def extract_zip(input_zip, output_dir: Path, strip_root: bool = False): shutil.copyfileobj(src, dst) kept += 1 kept_bytes += member.file_size - log(f"Extracted {kept} files ({kept_bytes / 1e9:.1f} GB); skipped {skipped} members " - f"(dirs {sorted(SKIP_DIRS)} + macOS cruft)") + log(f"Extracted {kept} files ({kept_bytes / 1e9:.1f} GB); skipped {skipped} members") + +############################################################# +# Decide whether the flat files need to be reconstructed # +############################################################# +# Reconstruct only when no flat files are available anywhere: a separate `input_flat_files` +# zip always means they exist (mouse brain); otherwise peek inside the raw zip for a +# `*_tx_file.csv`. Absent everywhere => the liver "Raw Data Files" case (Version 4). +if par["input_flat_files"]: + RECONSTRUCT = False +else: + log("No flat files zip provided; checking whether the raw zip contains flat files") + RECONSTRUCT = not _zip_has_flat_files(par["input_raw"]) +log(f"Reconstruct flat files from raw AnalysisResults: {RECONSTRUCT}") # Extract zip files log("Extract zip of raw files") INPUT_RAW_EXTRACTED = TMP_DIR / "input_raw" -extract_zip(par["input_raw"], INPUT_RAW_EXTRACTED, strip_root=True) +extract_zip( + par["input_raw"], + INPUT_RAW_EXTRACTED, + strip_root=True, + wanted=_member_wanted_reconstruct if RECONSTRUCT else _member_wanted, +) log(f"Files and folders in input_raw_extracted ({INPUT_RAW_EXTRACTED})") print(os.listdir(INPUT_RAW_EXTRACTED)) @@ -233,6 +296,124 @@ def extract_zip(input_zip, output_dir: Path, strip_root: bool = False): else: raise AssertionError(f"No CellLabels folder and no per-FOV CellLabels tifs found in {DATA_DIR}") +################################################################### +# Reconstruct the flat files from the raw decoding CSVs (Version 4)# +################################################################### +# See the module docstring. We rebuild the four flat files sopa needs +# (`{id}_fov_positions_file.csv`, `{id}_tx_file.csv`, `{id}_metadata_file.csv`, +# `{id}_exprMat_file.csv`) from AnalysisResults + RunSummary + Cell_Stats and drop them into +# DATA_DIR so the unchanged `sopa.io.cosmx` call below picks them up. We do NOT reconstruct +# `-polygons.csv` (the raw export has no cell boundaries); `cell_boundaries` is optional in +# the API and downstream methods produce their own segmentation. +# +# Coordinate reconstruction (validated end-to-end: 0.999 of assigned transcripts land on +# their own cell in sopa's stitched label image): +# scale = 1e3 / COSMX_PIXEL_SIZE (mm -> px, matching sopa's own fov-positions conversion) +# x_global_px = x_mm * scale + x_local +# y_global_px = y_mm * scale + (H - 1 - y_local) # sopa flips each label tile along y +# The same transform is applied to the Cell_Stats centroids so the table's obsm['spatial'] +# stays consistent with the transcripts and labels. +def reconstruct_flat_files(extracted_root: Path, data_dir: Path, labels_dir: Path, dataset_id: str) -> int: + import numpy as np + import pandas as pd + import tifffile + from sopa.io.reader.cosmx import COSMX_PIXEL_SIZE + + scale = 1e3 / COSMX_PIXEL_SIZE + + # FOV image height (all FOVs share the same shape; sopa asserts this later). + a_label = next(iter(sorted(labels_dir.glob("*.[Tt][Ii][Ff]")))) + with tifffile.TiffFile(a_label) as tif: + H, W = tif.pages[0].shape + log(f"Reconstruct: FOV image shape {(H, W)} (from {a_label.name})") + + # FOV positions from RunSummary/latest.fovs.csv (headerless: ..., x_mm[1], y_mm[2], ..., fov[6]). + fovs_files = sorted(extracted_root.rglob("*.fovs.csv")) + fovs_file = next((f for f in fovs_files if f.name == "latest.fovs.csv"), None) or fovs_files[0] + raw_pos = pd.read_csv(fovs_file, header=None) + fp = raw_pos[[6, 1, 2]].copy() + fp.columns = ["fov", "x_mm", "y_mm"] + fp = fp.drop_duplicates("fov").sort_values("fov").reset_index(drop=True) + fp.to_csv(data_dir / f"{dataset_id}_fov_positions_file.csv", index=False) + pos = fp.set_index("fov") + log(f"Reconstruct: {len(fp)} FOV positions from {fovs_file.name}") + + # Map fov number -> Cell_Stats csv (under CellStatsDir/FOV###/). + cs_paths = {} + for p in data_dir.rglob("*Cell_Stats_F*.csv"): + m = re.search(r"Cell_Stats_F(\d+)", p.name) + if m: + cs_paths[int(m.group(1))] = p + + cc_paths = sorted(extracted_root.rglob("*complete_code_cell_target_call_coord.csv")) + assert cc_paths, f"No complete_code_cell_target_call_coord.csv found under {extracted_root}" + log(f"Reconstruct: {len(cc_paths)} per-FOV transcript files, {len(cs_paths)} cell-stats files") + + tx_path = data_dir / f"{dataset_id}_tx_file.csv" + meta_parts, expr_parts = [], [] + max_cell_id = 0 + n_tx = 0 + with open(tx_path, "w") as tx_out: + for i, cc_path in enumerate(cc_paths): + m = re.search(r"FOV0*(\d+)", cc_path.name) or re.search(r"_F0*(\d+)", cc_path.name) + fov = int(m.group(1)) + if fov not in pos.index: + log(f"Reconstruct: WARNING no FOV position for FOV {fov}, skipping") + continue + xmm, ymm = float(pos.at[fov, "x_mm"]), float(pos.at[fov, "y_mm"]) + + cc = pd.read_csv(cc_path, usecols=["x", "y", "z", "target", "CellId"]) + tx = pd.DataFrame({ + "fov": fov, + "cell_ID": cc["CellId"].astype(int).values, + "target": cc["target"].astype(str).values, + "x_global_px": xmm * scale + cc["x"].values, + "y_global_px": ymm * scale + (H - 1 - cc["y"].values), + "z": cc["z"].values, + }) + tx.to_csv(tx_out, header=(i == 0), index=False) + n_tx += len(tx) + + # Per-cell metadata + counts, keyed on the segmentation cell list (Cell_Stats), + # so metadata and exprMat share the exact same cells in the same order. + cs = pd.read_csv(cs_paths[fov]) + cell_ids = cs["CellId"].astype(int).values + max_cell_id = max(max_cell_id, int(cell_ids.max()), int(tx["cell_ID"].max())) + + assigned = tx[tx["cell_ID"] > 0] + ct = pd.crosstab(assigned["cell_ID"], assigned["target"]).reindex(cell_ids, fill_value=0) + ct.insert(0, "cell_ID", cell_ids) + ct.insert(0, "fov", fov) + expr_parts.append(ct.reset_index(drop=True)) + + meta_parts.append(pd.DataFrame({ + "fov": fov, + "cell_ID": cell_ids, + "CenterX_global_px": xmm * scale + cs["CenterX"].values, + "CenterY_global_px": ymm * scale + (H - 1 - cs["CenterY"].values), + "Area": cs["Area"].values, + })) + + if (i + 1) % 50 == 0: + log(f"Reconstruct: processed {i + 1}/{len(cc_paths)} FOVs ({n_tx:,} transcripts)") + + meta = pd.concat(meta_parts, ignore_index=True) + meta.to_csv(data_dir / f"{dataset_id}_metadata_file.csv", index=False) + + expr = pd.concat(expr_parts, ignore_index=True).fillna(0) + gene_cols = [c for c in expr.columns if c not in ("fov", "cell_ID")] + expr[gene_cols] = expr[gene_cols].astype("int32") + expr = expr[["fov", "cell_ID"] + gene_cols] + expr.to_csv(data_dir / f"{dataset_id}_exprMat_file.csv", index=False) + + log(f"Reconstruct: wrote flat files — {n_tx:,} transcripts, {len(meta):,} cells, {len(gene_cols)} genes") + return max_cell_id + +RECON_MAX_CELL_ID = None +if RECONSTRUCT: + log("Reconstruct flat files from raw decoding CSVs") + RECON_MAX_CELL_ID = reconstruct_flat_files(INPUT_RAW_EXTRACTED, DATA_DIR, labels_dir, RECON_ID) + ############################################# # Memory optimization: lean transcript read # ############################################# @@ -286,6 +467,17 @@ def __getattr__(self, item): _cosmx_mod.pd = _PandasReadCsvProxy(_real_pd) +# When we reconstruct the flat files ourselves, pin sopa's global-cell-id formula to the +# segmentation's true max cell id. sopa otherwise derives that max separately from each of +# the tx / metadata / exprMat frames and asserts they agree — but a cell present in the +# segmentation (Cell_Stats) yet carrying zero transcripts makes the tx-file max smaller, +# tripping the assert. Using one fixed max keeps the ids consistent across all three files. +if RECONSTRUCT: + _RECON_MAX = int(RECON_MAX_CELL_ID) + def _fixed_global_cell_id(self, df): + return df["fov"] * (_RECON_MAX + 1) * (df["cell_ID"] > 0) + df["cell_ID"] + _cosmx_mod._CosMXReader._get_global_cell_id = _fixed_global_cell_id + ######################################### # Convert raw files to spatialdata zarr # ######################################### @@ -293,14 +485,16 @@ def __getattr__(self, item): log("Convert raw files to spatialdata zarr") sdata = sopa.io.cosmx( - DATA_DIR, - dataset_id=None, - fov=None, - read_proteins=False, - cells_labels=True, - cells_table=True, - cells_polygons=True, - flip_image=False + DATA_DIR, + dataset_id=RECON_ID if RECONSTRUCT else None, + fov=None, + read_proteins=False, + cells_labels=True, + cells_table=True, + # The reconstructed (liver) export has no cell-boundary polygons; every other CosMx + # export we handle ships `-polygons.csv`. + cells_polygons=not RECONSTRUCT, + flip_image=False, ) @@ -309,7 +503,7 @@ def __getattr__(self, item): ############### log("Rename keys") elements_renaming_map = { - "stitched_image" : "morphology_mip", + "stitched_image" : "morphology_mip", "stitched_labels" : "cell_labels", "points" : "transcripts", "cells_polygons" : "cell_boundaries", @@ -317,6 +511,10 @@ def __getattr__(self, item): } for old_key, new_key in elements_renaming_map.items(): + if old_key not in sdata: + # e.g. cells_polygons is absent when reconstructing (no boundaries in the raw export) + log(f"Element '{old_key}' not present, skipping rename to '{new_key}'") + continue sdata[new_key] = sdata[old_key] del sdata[old_key] @@ -338,6 +536,9 @@ def __getattr__(self, item): # Add info to metadata table # ############################## log("Add metadata to table") +# The common iST API requires a per-cell `cell_id` in obs; sopa keys the table by the global +# cell id (as the index) but doesn't add it as a column. +sdata["table"].obs["cell_id"] = sdata["table"].obs.index.astype(str) for key in ["dataset_id", "dataset_name", "dataset_url", "dataset_reference", "dataset_summary", "dataset_description", "dataset_organism", "segmentation_id"]: sdata["table"].uns[key] = par[key] @@ -348,4 +549,4 @@ def __getattr__(self, item): sdata.write(par["output"]) -log(f"Done in {datetime.now() - t0}") \ No newline at end of file +log(f"Done in {datetime.now() - t0}") diff --git a/src/datasets/loaders/vizgen_merscope/script.py b/src/datasets/loaders/vizgen_merscope/script.py index 5bf8e8182..d3251cb95 100644 --- a/src/datasets/loaders/vizgen_merscope/script.py +++ b/src/datasets/loaders/vizgen_merscope/script.py @@ -4,8 +4,10 @@ from datetime import datetime from pathlib import Path +import dask.array as da import geopandas as gpd import h5py +import numpy as np import pandas as pd import spatialdata as sd import spatialdata_io as sdio @@ -44,47 +46,50 @@ ) +def _cell_polygon(cell_grp): + """Boundary for one cell: prefer zIndex_3, else any available z-plane. + + MERSCOPE stores a cell's boundary only at the z-planes where it appears, so keying on a + single hardcoded z-index (the old ``["zIndex_3"]``) drops -- or KeyErrors on -- every cell + not present at that plane. Returns None if the cell has no usable boundary. + """ + z_keys = list(cell_grp.keys()) + z = "zIndex_3" if "zIndex_3" in z_keys else (z_keys[0] if z_keys else None) + if z is None or "p_0" not in cell_grp[z] or "coordinates" not in cell_grp[z]["p_0"]: + return None + return MultiPolygon([Polygon(cell_grp[z]["p_0"]["coordinates"][()][0])]) + + def read_boundary_hdf5(folder): print(datetime.now() - t0, "Convert boundary hdf5 to parquet", flush=True) - all_boundaries = {} - boundaries = None - hdf5_files = os.listdir(folder + "/cell_boundaries/") + hdf5_files = [ + f for f in os.listdir(folder + "/cell_boundaries/") if f.endswith(".hdf5") + ] n_files = len(hdf5_files) - incr = n_files // 15 - for _, i in enumerate(hdf5_files): - if (_ % incr) == 0: - print(datetime.now() - t0, f"\tProcessed {_}/{n_files}", flush=True) - with h5py.File(folder + "/cell_boundaries/" + i, "r") as f: - for key in f["featuredata"].keys(): - if boundaries is not None: - boundaries.loc[key] = MultiPolygon( - [ - Polygon( - f["featuredata"][key]["zIndex_3"]["p_0"]["coordinates"][ - () - ][0] - ) - ] - ) # doesn't matter which zIndex we use, MultiPolygon to work with read function in spatialdata-io - else: - # boundaries = gpd.GeoDataFrame(index=[key], geometry=MultiPolygon([Polygon(f['featuredata'][key]['zIndex_3']['p_0']['coordinates'][()][0])])) - boundaries = gpd.GeoDataFrame( - geometry=gpd.GeoSeries( - MultiPolygon( - [ - Polygon( - f["featuredata"][key]["zIndex_3"]["p_0"][ - "coordinates" - ][()][0] - ) - ] - ), - index=[key], - ) - ) - all_boundaries[i] = boundaries - boundaries = None - all_concat = pd.concat(list(all_boundaries.values())) + incr = max(1, n_files // 15) # guard against ZeroDivision when <15 files + geoms, index, n_skipped = [], [], 0 + for n, fname in enumerate(hdf5_files): + if n % incr == 0: + print(datetime.now() - t0, f"\tProcessed {n}/{n_files}", flush=True) + with h5py.File(folder + "/cell_boundaries/" + fname, "r") as f: + featuredata = f["featuredata"] + for key in featuredata.keys(): + poly = _cell_polygon(featuredata[key]) + if poly is None: + n_skipped += 1 + continue + geoms.append(poly) + index.append(key) + # n_skipped is logged (never silently dropped) so a re-run reveals whether low boundary + # coverage is a reader problem or a genuine property of the raw hdf5. + print( + datetime.now() - t0, + f"Read {len(geoms)} cell boundaries from {n_files} files " + f"({n_skipped} cells had no usable z-plane boundary)", + flush=True, + ) + # Build the GeoDataFrame in one pass; the old row-by-row `.loc` growth was O(n^2) over ~10^5 cells. + all_concat = gpd.GeoDataFrame(geometry=gpd.GeoSeries(geoms, index=index)) all_concat = all_concat[ ~all_concat.index.duplicated(keep="first") ] # hdf5 can contain duplicates with same cell_id and position, removing those @@ -241,23 +246,48 @@ def read_boundary_hdf5(folder): "return_regions_as_labels": True, } -for i in range(n_iter): - print( - datetime.now() - t0, - f"Rasterizing iteration {i + 1}/{n_iter} (cells {i * N}..{min((i + 1) * N, n_cells)})", - flush=True, +if n_iter == 1: + # Common case (every current dataset has <65535 cells): a single rasterize call. + # Keep the lazy DataArray exactly as before. + print(datetime.now() - t0, "Rasterizing all cells in a single pass", flush=True) + labels_image = sd.rasterize( + sdata["cell_boundaries"], ["x", "y"], **rasterize_args ) - labels_image_ = sd.rasterize( - sdata["cell_boundaries"].iloc[i * N : min((i + 1) * N, n_cells)], - ["x", "y"], - **rasterize_args, +else: + # >65535 cells: rasterize in chunks and merge. Two subtleties make the naive in-place + # merge wrong (it was silently broken before; this path had never run on real data since + # no dataset exceeds 65535 cells): + # 1. sd.rasterize returns a *lazy* (dask-backed) DataArray whose `.values` is a computed + # copy, so the old `labels_image.values[mask] = ...` write never persisted. We + # materialise each chunk and merge in a plain numpy array instead. + # 2. return_regions_as_labels labels each chunk 1..len(chunk) *positionally*, so chunks + # would collide on the same label. We offset each chunk by its global start index + # (and use uint32, since labels exceed uint16 past 65535 cells). + combined = None + template = None + for i in range(n_iter): + start = i * N + end = min((i + 1) * N, n_cells) + print( + datetime.now() - t0, + f"Rasterizing iteration {i + 1}/{n_iter} (cells {start}..{end})", + flush=True, + ) + chunk = sd.rasterize( + sdata["cell_boundaries"].iloc[start:end], ["x", "y"], **rasterize_args + ) + chunk_np = np.asarray(chunk.data) + if combined is None: + combined = chunk_np.astype("uint32") + template = chunk + else: + mask = chunk_np > 0 + combined[mask] = chunk_np[mask].astype("uint32") + start + # Rebuild a dask-backed labels element (spatialdata rejects a numpy-backed one) while + # preserving the template's transform and attrs (incl. label_index_to_category). + labels_image = template.copy( + data=da.from_array(combined, chunks=template.data.chunksize) ) - if i == 0: - labels_image = labels_image_ - else: - labels_image.values[labels_image_.values > 0] = labels_image_.values[ - labels_image_.values > 0 - ] print(datetime.now() - t0, "Rasterization finished", flush=True) try: diff --git a/src/methods_segmentation/binning/config.vsh.yaml b/src/methods_segmentation/binning/config.vsh.yaml index c9474b430..5bc586957 100644 --- a/src/methods_segmentation/binning/config.vsh.yaml +++ b/src/methods_segmentation/binning/config.vsh.yaml @@ -14,6 +14,20 @@ arguments: - name: --bin_size type: integer default: 30 + description: | + Edge length of each square pseudo-cell bin, in PIXELS of the input `image` + element. On this benchmark's standardized raw_ist grid (images are rasterized + at target_unit_to_pixels=1, i.e. ~1 um/pixel) 1 px is ~1 um, so 30 px is + ~30 um bins. Used only when --bin_size_um is not set. + - name: --bin_size_um + type: double + required: false + description: | + Edge length of each square bin in MICRONS. When set, it OVERRIDES --bin_size: + the script reads the input image's coordinate transform (um per pixel) and + converts to a pixel bin size, so the physical bin size is comparable across + datasets even if they are gridded at different pixel sizes. Leave unset to use + --bin_size (raw pixels). resources: - type: python_script diff --git a/src/methods_segmentation/binning/script.py b/src/methods_segmentation/binning/script.py index 49ead7e0b..ec8255600 100644 --- a/src/methods_segmentation/binning/script.py +++ b/src/methods_segmentation/binning/script.py @@ -24,10 +24,36 @@ def convert_to_lower_dtype(arr): return arr.astype(new_dtype) + +def pixel_size_um(image_element, coordinate_system="global"): + """Microns-per-pixel of an image element, read from its coordinate transform. + + On this benchmark's standardized raw_ist grid the images are rasterized at + target_unit_to_pixels=1 (~1 um/pixel), so this is ~1.0 for current datasets; + reading it keeps --bin_size_um correct if a dataset is ever gridded at a + different resolution. Falls back to 1.0 um/px (the standardized grid) if the + transform cannot be read, so a micron sweep still runs on current data. + """ + try: + transforms = image_element.transform # dict: coord_system -> transformation + trans = transforms.get(coordinate_system) or next(iter(transforms.values())) + aff = trans.to_affine_matrix(input_axes=("y", "x"), output_axes=("y", "x")) + sy, sx = abs(float(aff[0, 0])), abs(float(aff[1, 1])) + px = (sy + sx) / 2.0 + if not np.isfinite(px) or px <= 0: + raise ValueError(f"non-positive pixel size {px}") + return px + except Exception as e: + print(f"WARNING: could not read pixel size from transform ({e}); " + f"assuming 1.0 um/pixel (the standardized raw_ist grid).", flush=True) + return 1.0 + ## VIASH START par = { "input": "../task_ist_preprocessing/resources_test/common/2023_10x_mouse_brain_xenium/dataset.zarr", - "output": "segmentation.zarr" + "output": "segmentation.zarr", + "bin_size": 30, + "bin_size_um": None } ## VIASH END @@ -43,7 +69,19 @@ def convert_to_lower_dtype(arr): sd_output = sd.SpatialData() image = sdata['image']['scale0'].image.compute().to_numpy() transformation = sdata['image']['scale0'].image.transform.copy() -img_arr = tx.preprocessing.segment_binning(image[0], hyperparameters['bin_size']) ### TOdo find the optimal bin_size + +# bin_size is in PIXELS of the image grid. If bin_size_um is given, it overrides +# bin_size: convert the physical (micron) bin edge to pixels via the image's +# um-per-pixel scale, so the bin size is comparable across datasets. +if hyperparameters.get('bin_size_um') is not None: + px_um = pixel_size_um(sdata['image']['scale0'].image) + bin_size = max(1, int(round(float(hyperparameters['bin_size_um']) / px_um))) + print(f"bin_size_um={hyperparameters['bin_size_um']} um / {px_um} um/px " + f"-> bin_size={bin_size} px", flush=True) +else: + bin_size = int(hyperparameters['bin_size']) + +img_arr = tx.preprocessing.segment_binning(image[0], bin_size) image = convert_to_lower_dtype(img_arr) data_array = xr.DataArray(image, name=f'segmentation', dims=('y', 'x')) parsed_data = Labels2DModel.parse(data_array, transformations=transformation) diff --git a/src/methods_segmentation/cellpose/config.vsh.yaml b/src/methods_segmentation/cellpose/config.vsh.yaml index 4f982f3ed..57b6516ea 100644 --- a/src/methods_segmentation/cellpose/config.vsh.yaml +++ b/src/methods_segmentation/cellpose/config.vsh.yaml @@ -69,6 +69,14 @@ arguments: - name: --min_size type: integer default: 15 + - name: --niter + type: integer + default: 0 + description: | + Number of flow-dynamics iterations. 0 (Cellpose treats 0 or None the same) + lets Cellpose set it proportional to the diameter (~200 at resample=true). + Lower it (e.g. 50) to speed up the CPU path; raise it for large/elongated + cells that under-converge. Forwarded verbatim to model.eval. - name: --stitch_threshold type: double default: 0.0 @@ -84,13 +92,16 @@ engines: setup: - type: python pypi: cellpose<4.0.0 - # Pre-download the pretrained cyto weights into the image (~/.cellpose) - # so runtime tasks never hit the cellpose model server (avoids the - # intermittent "HTTP Error 504: Gateway Time-out" seen when several - # segmentation tasks fetch the weights concurrently). + # Pre-download the pretrained weights + size models into the image + # (~/.cellpose) so runtime tasks never hit the cellpose model server + # (avoids the intermittent "HTTP Error 504: Gateway Time-out" seen when + # several segmentation tasks fetch the weights concurrently). Instantiating + # Cellpose(model_type=m) caches both the mask model and its size model. + # All four builtins are cached because the --model_type sweep selects among + # them (see scripts/run_benchmark/cellpose_params.yaml). - type: docker run: - - "python -c \"from cellpose import models; models.Cellpose(gpu=False, model_type='cyto')\"" + - "python -c \"from cellpose import models; [models.Cellpose(gpu=False, model_type=m) for m in ['cyto','nuclei','cyto2','cyto3']]\"" __merge__: - /src/base/setup_txsim_partial.yaml - /src/base/setup_spatialdata_partial.yaml diff --git a/src/methods_segmentation/stardist/config.vsh.yaml b/src/methods_segmentation/stardist/config.vsh.yaml index 0330b270a..cba9c2272 100644 --- a/src/methods_segmentation/stardist/config.vsh.yaml +++ b/src/methods_segmentation/stardist/config.vsh.yaml @@ -15,6 +15,38 @@ arguments: - name: --model type: string default: "2D_versatile_fluo" + description: >- + Pretrained StarDist2D model loaded via StarDist2D.from_pretrained. + Fluorescence/nuclear models: "2D_versatile_fluo" (default, trained on a DSB2018 + subset) and "2D_paper_dsb2018". "2D_versatile_he" is a brightfield H&E RGB model + and is NOT appropriate for the single-channel fluorescence morphology images here. + # --- StarDist predict_instances knobs, forwarded through predict_instances_big --- + # All optional: when unset the model's own optimized values are used (thresholds.json + # for prob/nms, no rescaling for scale) — i.e. exactly the pre-tuning behaviour. + - name: --prob_thresh + type: double + required: false + description: >- + Object probability threshold; candidate pixels below it are discarded. If unset, + the model's optimized value is used (0.479071 for 2D_versatile_fluo). LOWER => more / + dimmer nuclei detected (higher recall); HIGHER => fewer, more confident detections + (higher precision). Valid range ~0..1. + - name: --nms_thresh + type: double + required: false + description: >- + Non-maximum-suppression IoU threshold for overlapping polygon candidates. If unset, + the model's optimized value is used (0.3 for 2D_versatile_fluo). LOWER => more + aggressive suppression (touching nuclei more likely merged/dropped); HIGHER => keeps + more overlapping detections. Valid range ~0..1. + - name: --scale + type: double + required: false + description: >- + Isotropic factor the image is rescaled by before prediction and undone afterwards. + If unset, no rescaling. Use it to match nucleus size to the model's training size: + >1 upscales (small nuclei appear larger), <1 downscales. StarDist's analogue of + Cellpose's diameter. resources: - type: python_script diff --git a/src/methods_segmentation/stardist/script.py b/src/methods_segmentation/stardist/script.py index 82d72ec4f..4bf502c61 100644 --- a/src/methods_segmentation/stardist/script.py +++ b/src/methods_segmentation/stardist/script.py @@ -39,7 +39,10 @@ def convert_to_lower_dtype(arr): par = { "input": "resources_test/task_ist_preprocessing/mouse_brain_combined/raw_ist.zarr", "output": "temp/stardist/segmentation.zarr", - "model": "2D_versatile_fluo" + "model": "2D_versatile_fluo", + "prob_thresh": None, + "nms_thresh": None, + "scale": None, } ## VIASH END @@ -75,8 +78,20 @@ def do_after(self): block_size = min(image.shape[1] // 3, 4096) offset = min(block_size // 5.5, 128) +# Tunable knobs forwarded through predict_instances_big -> predict_instances. +# A value left as None (i.e. omitted from par) means "use the model's own optimized +# value": thresholds.json for prob_thresh/nms_thresh, no rescaling for scale. This +# keeps the default (no-args) call identical to the pre-tuning behaviour. +eval_params = { + k: par[k] + for k in ("prob_thresh", "nms_thresh", "scale") + if par.get(k) is not None +} +print(f"predict_instances_big overrides: {eval_params}", flush=True) + labels, _ = model.predict_instances_big( - image[0,:,:], axes='YX', block_size=block_size, min_overlap=offset, context=offset, normalizer=normalizer#, n_tiles=(4,4) + image[0,:,:], axes='YX', block_size=block_size, min_overlap=offset, + context=offset, normalizer=normalizer, **eval_params # n_tiles left to block_size ) diff --git a/src/methods_segmentation/watershed/config.vsh.yaml b/src/methods_segmentation/watershed/config.vsh.yaml index cd554a435..54b25f622 100644 --- a/src/methods_segmentation/watershed/config.vsh.yaml +++ b/src/methods_segmentation/watershed/config.vsh.yaml @@ -156,6 +156,10 @@ arguments: - name: --bg_intensity_filter_bg_size type: integer default: 2000 +- name: --watershed_compactness + type: double + default: 0.0 + description: "Compactness passed to skimage.segmentation.watershed. 0.0 = standard marker-controlled watershed (unchanged default). Higher values (try 1e-3 to 1e-1) enable compact watershed (Neubert & Protzel), biasing basins toward rounder, more regular cell shapes. Forwarded via txsim's watershed_params dict (see script.py)." resources: - type: python_script diff --git a/src/methods_segmentation/watershed/script.py b/src/methods_segmentation/watershed/script.py index 198e30b3a..f52fbf05c 100644 --- a/src/methods_segmentation/watershed/script.py +++ b/src/methods_segmentation/watershed/script.py @@ -37,6 +37,14 @@ def convert_to_lower_dtype(arr): del hyperparameters['input'] del hyperparameters['output'] +# skimage.segmentation.watershed's `compactness` knob is only reachable through the +# nested `watershed_params` dict that txsim.segment_watershed forwards (it reads +# hyperparams.get("watershed_params", {})). Lift the exposed --watershed_compactness +# arg into that dict so it actually takes effect. compactness=0.0 == standard +# marker-controlled watershed, i.e. unchanged default behaviour. +compactness = hyperparameters.pop("watershed_compactness", 0.0) +hyperparameters["watershed_params"] = {"compactness": compactness} + sdata = sd.read_zarr(par["input"]) sd_output = sd.SpatialData() diff --git a/src/methods_transcript_assignment/fastreseg/config.vsh.yaml b/src/methods_transcript_assignment/fastreseg/config.vsh.yaml index d5c9985ec..33e175e51 100644 --- a/src/methods_transcript_assignment/fastreseg/config.vsh.yaml +++ b/src/methods_transcript_assignment/fastreseg/config.vsh.yaml @@ -78,12 +78,10 @@ engines: run: - Rscript -e 'options(warn = 2); remotes::install_github("Nanostring-Biostats/FastReseg", upgrade = "never", repos = "https://cran.rstudio.com")' - type: python - # `plankton` (pulled in by txsim.run_ssam for ssam cell annotation in - # input.py) still does `from matplotlib.cm import get_cmap`, removed in - # matplotlib 3.9. Pin < 3.9 so run_ssam imports. This is a local setup - # step, so it runs AFTER the __merge__'d spatialdata/txsim partials (which - # otherwise pull matplotlib >= 3.9) and wins. Mirrors methods_cell_type_annotation/ssam. - pypi: [planktonspace, "matplotlib<3.9"] + # tacco provides the cell-type annotation used in input.py (tacco.tl.annotate). + # Mirrors methods_cell_type_annotation/tacco. This replaced the previous ssam + # path, which needed planktonspace + a matplotlib<3.9 pin. + pypi: [anndata, numpy, tacco] __merge__: - /src/base/setup_txsim_partial.yaml - /src/base/setup_spatialdata_partial.yaml diff --git a/src/methods_transcript_assignment/fastreseg/input.py b/src/methods_transcript_assignment/fastreseg/input.py index 9eabfe0ab..207952ee2 100644 --- a/src/methods_transcript_assignment/fastreseg/input.py +++ b/src/methods_transcript_assignment/fastreseg/input.py @@ -4,7 +4,7 @@ import pandas as pd import xarray as xr import dask -import txsim as tx +import tacco import anndata as ad import argparse from scipy.sparse import csr_matrix @@ -90,12 +90,31 @@ def parse_arguments(): output_path_cell_type = args.output_path_cell_type ## potential other parameters (TODO - make configurable) -um_per_pixel = 0.5 sc_celltype_key = 'cell_type' ### reading the data in sdata = sd.read_zarr(input_path) +# The transcripts points can carry a non-unique index: multi-partition parquet where +# each partition's RangeIndex restarts at 0, or concatenated "combined" replicates that +# each kept their own row index. dask-expr aligns on the index when the cell_id column is +# assigned below (`sdata['transcripts']["cell_id"] = ...`), and pandas cannot reindex a +# duplicate-labelled axis -> the following sd.transform then fails with +# "cannot reindex on an axis with duplicate labels". Reset to a unique RangeIndex eagerly +# here: a lazy .reset_index() does not help (the duplicate index already poisons the dask +# graph), and re-parsing via PointsModel.parse breaks the from_dask_array(index=...) assign +# below with a "Missing dependency" graph error, so materialise to pandas and re-wrap as a +# single from_pandas partition, re-attaching the coordinate transformations. No-op when the +# index is already unique. +if not sdata['transcripts'].index.compute().is_unique: + print('Transcripts index has duplicate labels; resetting to a unique index', flush=True) + _transforms = sd.transformations.get_transformation(sdata['transcripts'], get_all=True) + _fixed = dask.dataframe.from_pandas( + sdata['transcripts'].compute().reset_index(drop=True), npartitions=1 + ) + sd.transformations.set_transformation(_fixed, _transforms, set_all=True) + sdata['transcripts'] = _fixed + ### reading in basic segmentation sdata_segm = sd.read_zarr(input_segmentation_path) segmentation_coord_systems = sd.transformations.get_transformation(sdata_segm["segmentation"], get_all=True).keys() @@ -125,10 +144,10 @@ def parse_arguments(): print('Transforming transcripts coordinates', flush=True) transcripts = sd.transform(sdata['transcripts'], to_coordinate_system='global') -# .copy() is required: for a single-partition points object, transcripts.compute() -# returns a reference to the dask graph's backing frame, so an in-place rename here -# would corrupt it and the later transcripts.compute() (passed to run_ssam) would -# yield renamed columns that no longer match the dask _meta -> "Metadata mismatch". +# .copy() defensively: for a single-partition points object, transcripts.compute() +# can return a reference to the dask graph's backing frame, so the in-place rename +# below would otherwise corrupt it and any later compute of `transcripts` would yield +# renamed columns that no longer match the dask _meta -> "Metadata mismatch". transcripts_df = transcripts.compute().copy() transcripts_df.rename(columns = {'feature_name': 'target', 'transcript_id': 'UMI_transID', 'cell_id': 'UMI_cellID'}, inplace = True) @@ -157,18 +176,31 @@ def parse_arguments(): columns=adata_sp.var_names) count_df.to_csv(output_path_counts) -#### run cell annotation with ssam +#### run cell annotation with tacco adata_sc = ad.read_h5ad(input_sc_reference_path) -adata_sc.X = adata_sc.layers["normalized"] print(adata_sc.var_names[1:10]) -shared_genes = [g for g in adata_sc.var_names if g in adata_sp.var_names] -adata_sp = adata_sp[:,shared_genes] +# tacco annotates the cell x gene count matrix directly against the reference, so +# (unlike the previous ssam path) it needs neither the transcript-level spots nor a +# gene subsetting step -- tacco intersects genes internally. It expects raw counts on +# .X for both spatial and reference, mirroring methods_cell_type_annotation/tacco. +adata_sp.X = adata_sp.layers['counts'] +adata_sc.X = adata_sc.layers['counts'] print('Annotating cell types', flush=True) -adata_sp = tx.preprocessing.run_ssam( - adata_sp, transcripts.compute(), adata_sc, um_p_px=um_per_pixel, - cell_id_col='cell_id', gene_col='feature_name', sc_ct_key=sc_celltype_key +cell_type_assignment = tacco.tl.annotate( + adata=adata_sp, + reference=adata_sc, + annotation_key=sc_celltype_key, ) -cell_type_df = adata_sp.obs["ct_ssam"].astype(str) + +# tacco returns an n_obs x n_celltypes proportion matrix (aligned to adata_sp.obs); +# take the argmax per cell for the hard label the FastReseg R step consumes as `clust`. +cell_types = cell_type_assignment.columns +highest_score_idx = np.argmax(cell_type_assignment.to_numpy(), axis=1) +adata_sp.obs[sc_celltype_key] = cell_types[highest_score_idx] + +# Preserve the exact CSV shape the R step reads (read.csv(row.names=1) -> a single +# column of per-cell labels indexed by cell_id). +cell_type_df = adata_sp.obs[sc_celltype_key].astype(str) cell_type_df.to_csv(output_path_cell_type, header=True) \ No newline at end of file diff --git a/src/methods_transcript_assignment/fastreseg/script.R b/src/methods_transcript_assignment/fastreseg/script.R index f1e67810e..bc864d170 100644 --- a/src/methods_transcript_assignment/fastreseg/script.R +++ b/src/methods_transcript_assignment/fastreseg/script.R @@ -81,7 +81,9 @@ if(is.null(refineAll_res_one_FOC$updated_transDF_list[[1]])){ cell_df$UMI_cellID <- as.integer(row.names(cell_df)) transcriptDF <- left_join(cell_df, transcriptDF) names(transcriptDF)[names(transcriptDF) == "UMI_cellID"] <- "updated_cellID" - names(transcriptDF)[names(transcriptDF) == "ct_ssam"] <- "updated_celltype" + # input.py writes the per-cell annotation column under sc_celltype_key ("cell_type"). + # (Was "ct_ssam" when the annotation step used ssam.) + names(transcriptDF)[names(transcriptDF) == "cell_type"] <- "updated_celltype" write.csv(transcriptDF, file = path_to_transcripts_out) } else { write.csv(refineAll_res_one_FOC$updated_transDF_list[[1]], file = path_to_transcripts_out) diff --git a/src/methods_transcript_assignment/pciseq/script.py b/src/methods_transcript_assignment/pciseq/script.py index 1156a833d..264c61718 100644 --- a/src/methods_transcript_assignment/pciseq/script.py +++ b/src/methods_transcript_assignment/pciseq/script.py @@ -142,7 +142,7 @@ def eta_update_no_assert(self): # Fail fast with an actionable message instead. This happens when the upstream `cell_labels` # copied by custom_segmentation is empty (observed for the vizgen lung-cancer merscope dataset, # whose polygon->label rasterization produced an all-zero image). -if not (label_image > 0).any(): +if label_image.max() == 0: raise ValueError( f"Segmentation '{par['input_segmentation']}' contains no cells: the label image " f"(shape={label_image.shape}, dtype={label_image.dtype}) is entirely zero. pciSeq " diff --git a/src/methods_transcript_assignment/segger/config.vsh.yaml b/src/methods_transcript_assignment/segger/config.vsh.yaml index f83d61b0e..80130ce0d 100644 --- a/src/methods_transcript_assignment/segger/config.vsh.yaml +++ b/src/methods_transcript_assignment/segger/config.vsh.yaml @@ -141,4 +141,4 @@ runners: - type: executable - type: nextflow directives: - label: [ hightime, midcpu, highmem, gpu ] + label: [ hightime, midcpu, highmem, gpuh100 ]