Fixes

scripts/create_resources/spatial/mirror_bruker_to_s3.sh

@@ -23,17 +23,30 @@ set -euo pipefail
 # --- Config -----------------------------------------------------------------
 
 SRC_BASE="https://smi-public.objects.liquidweb.services"
-S3_DEST="${S3_DEST:-s3://openproblems-data/resources_raw/bruker_cosmx}"
+# NSCLC lung-cancer samples live on a different NanoString host, one folder per sample.
+NSCLC_BASE="https://nanostring-public-share.s3.us-west-2.amazonaws.com/SMI-Compressed"
+# Where the loaders read from (see the process_bruker_cosmx*.sh run scripts).
+S3_DEST="${S3_DEST:-s3://openproblems-data/resources/raw_data/bruker_cosmx}"
 SCRATCH_DIR="${SCRATCH_DIR:-$PWD/bruker_mirror_scratch}"
 
-# Files to mirror. The URL-encoded names are what the server serves; the second
-# column is the (decoded) name to store under on S3.
+# Files to mirror. Each entry is "SOURCE|LOCAL_NAME":
+#   - SOURCE is either a full https URL, or a name relative to SRC_BASE (the mouse/liver host).
+#     The URL-encoded names are what the liquidweb server serves.
+#   - LOCAL_NAME is the (decoded) name to store under on S3.
 FILES=(
   "HalfBrain.zip|HalfBrain.zip"
   "Half%20%20Brain%20simple%20%20files%20.zip|Half Brain simple files.zip"
   "NormalLiverFiles.zip|NormalLiverFiles.zip"
 )
 
+# NSCLC lung-cancer samples: each ships a flat-files+cell-labels archive and a
+# raw-morphology-images archive. The bruker_cosmx_nsclc loader streams both from S3.
+NSCLC_SAMPLES=(Lung5_Rep1 Lung5_Rep2 Lung5_Rep3 Lung6 Lung9_Rep1 Lung9_Rep2 Lung12 Lung13)
+for s in "${NSCLC_SAMPLES[@]}"; do
+  FILES+=("$NSCLC_BASE/$s/${s}+SMI+Flat+data.tar.gz|${s}+SMI+Flat+data.tar.gz")
+  FILES+=("$NSCLC_BASE/$s/${s}+RawMorphologyImages.tar.gz|${s}+RawMorphologyImages.tar.gz")
+done
+
 # --- Preflight --------------------------------------------------------------
 
 command -v curl >/dev/null || { echo "ERROR: curl not found" >&2; exit 1; }
@@ -64,7 +77,11 @@ s3_size() {
 for entry in "${FILES[@]}"; do
   url_name="${entry%%|*}"
   local_name="${entry##*|}"
-  url="$SRC_BASE/$url_name"
+  if [[ "$url_name" == http*://* ]]; then
+    url="$url_name"
+  else
+    url="$SRC_BASE/$url_name"
+  fi
   local_path="$SCRATCH_DIR/$local_name"
 
   # S3 key = everything after s3://bucket/
@@ -126,9 +143,11 @@ done
 echo "================================================================"
 echo "All files mirrored to $S3_DEST"
 echo
-echo "Next: update the input_raw / input_flat_files URLs in"
-echo "  scripts/create_resources/spatial/process_bruker_cosmx_nebius.sh"
-echo "to point at the S3 paths, e.g.:"
-echo "  input_raw:        $S3_DEST/HalfBrain.zip"
-echo "  input_flat_files: $S3_DEST/Half Brain simple files.zip"
-echo "  input_raw:        $S3_DEST/NormalLiverFiles.zip"
+echo "Next: the process_bruker_cosmx*.sh run scripts already point at these S3 paths, e.g.:"
+echo "  mouse/liver (process_bruker_cosmx_nebius.sh):"
+echo "    input_raw:        $S3_DEST/HalfBrain.zip"
+echo "    input_flat_files: $S3_DEST/Half Brain simple files.zip"
+echo "    input_raw:        $S3_DEST/NormalLiverFiles.zip"
+echo "  nsclc (process_bruker_cosmx_nsclc_nebius.sh), per sample:"
+echo "    input_raw:        $S3_DEST/<sample>+SMI+Flat+data.tar.gz"
+echo "    input_morphology: $S3_DEST/<sample>+RawMorphologyImages.tar.gz"

scripts/create_resources/spatial/process_bruker_cosmx_nsclc.sh

@@ -10,11 +10,16 @@ set -e
 
 publish_dir="s3://openproblems-data/resources/datasets"
 
+# Raw NSCLC archives mirrored to S3 (see scripts/create_resources/spatial/mirror_bruker_to_s3.sh).
+# Each sample ships two tar.gz archives: the flat files + cell labels, and the raw morphology images.
+raw_dir="s3://openproblems-data/resources/raw_data/bruker_cosmx"
+
 cat > /tmp/params.yaml << HERE
 param_list:
 
   - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung5_rep1"
-    sample: "Lung5_Rep1"
+    input_raw: "$raw_dir/Lung5_Rep1+SMI+Flat+data.tar.gz"
+    input_morphology: "$raw_dir/Lung5_Rep1+RawMorphologyImages.tar.gz"
     dataset_name: "Bruker CosMx Human Lung Cancer Lung5 Rep1"
     dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/"
     dataset_summary: "Bruker CosMx Human Lung Cancer Lung5 Rep1 dataset on FFPE."
@@ -23,16 +28,18 @@ param_list:
     segmentation_id: ["cell"]
 
   - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung5_rep2"
-    sample: "Lung5_Rep2"
+    input_raw: "$raw_dir/Lung5_Rep2+SMI+Flat+data.tar.gz"
+    input_morphology: "$raw_dir/Lung5_Rep2+RawMorphologyImages.tar.gz"
     dataset_name: "Bruker CosMx Human Lung Cancer Lung5 Rep2"
     dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/"
     dataset_summary: "Bruker CosMx Human Lung Cancer Lung5 Rep2 dataset on FFPE."
     dataset_description: "Bruker CosMx Human Lung Cancer Lung5 Rep2 dataset on FFPE. Adenocarcinoma, G1, T2aN2M0, IIIA, 75% tumour content."
     dataset_organism: "homo_sapiens"
     segmentation_id: ["cell"]
-    
+
   - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung5_rep3"
-    sample: "Lung5_Rep3"
+    input_raw: "$raw_dir/Lung5_Rep3+SMI+Flat+data.tar.gz"
+    input_morphology: "$raw_dir/Lung5_Rep3+RawMorphologyImages.tar.gz"
     dataset_name: "Bruker CosMx Human Lung Cancer Lung5 Rep3"
     dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/"
     dataset_summary: "Bruker CosMx Human Lung Cancer Lung5 Rep3 dataset on FFPE."
@@ -41,7 +48,8 @@ param_list:
     segmentation_id: ["cell"]
 
   - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung6"
-    sample: "Lung6"
+    input_raw: "$raw_dir/Lung6+SMI+Flat+data.tar.gz"
+    input_morphology: "$raw_dir/Lung6+RawMorphologyImages.tar.gz"
     dataset_name: "Bruker CosMx Human Lung Cancer Lung6"
     dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/"
     dataset_summary: "Bruker CosMx Human Lung Cancer Lung6 dataset on FFPE."
@@ -50,7 +58,8 @@ param_list:
     segmentation_id: ["cell"]
 
   - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung9_rep1"
-    sample: "Lung9_Rep1"
+    input_raw: "$raw_dir/Lung9_Rep1+SMI+Flat+data.tar.gz"
+    input_morphology: "$raw_dir/Lung9_Rep1+RawMorphologyImages.tar.gz"
     dataset_name: "Bruker CosMx Human Lung Cancer Lung9 Rep1"
     dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/"
     dataset_summary: "Bruker CosMx Human Lung Cancer Lung9 Rep1 dataset on FFPE."
@@ -59,7 +68,8 @@ param_list:
     segmentation_id: ["cell"]
 
   - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung9_rep2"
-    sample: "Lung9_Rep2"
+    input_raw: "$raw_dir/Lung9_Rep2+SMI+Flat+data.tar.gz"
+    input_morphology: "$raw_dir/Lung9_Rep2+RawMorphologyImages.tar.gz"
     dataset_name: "Bruker CosMx Human Lung Cancer Lung9 Rep2"
     dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/"
     dataset_summary: "Bruker CosMx Human Lung Cancer Lung9 Rep2 dataset on FFPE."
@@ -68,7 +78,8 @@ param_list:
     segmentation_id: ["cell"]
 
   - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung12"
-    sample: "Lung12"
+    input_raw: "$raw_dir/Lung12+SMI+Flat+data.tar.gz"
+    input_morphology: "$raw_dir/Lung12+RawMorphologyImages.tar.gz"
     dataset_name: "Bruker CosMx Human Lung Cancer Lung12"
     dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/"
     dataset_summary: "Bruker CosMx Human Lung Cancer Lung12 dataset on FFPE."
@@ -77,7 +88,8 @@ param_list:
     segmentation_id: ["cell"]
 
   - id: "bruker_cosmx/bruker_human_lung_cancer_cosmx/lung13"
-    sample: "Lung13"
+    input_raw: "$raw_dir/Lung13+SMI+Flat+data.tar.gz"
+    input_morphology: "$raw_dir/Lung13+RawMorphologyImages.tar.gz"
     dataset_name: "Bruker CosMx Human Lung Cancer Lung13"
     dataset_url: "https://nanostring.com/products/cosmx-spatial-molecular-imager/ffpe-dataset/nsclc-ffpe-dataset/"
     dataset_summary: "Bruker CosMx Human Lung Cancer Lung13 dataset on FFPE."
@@ -98,4 +110,4 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
   --workspace 53907369739130 \
   --params-file /tmp/params.yaml \
   --config common/nextflow_helpers/labels_tw.config \
-  --labels datasets,bruker_cosmx_nsclc
+  --labels datasets,bruker_cosmx_nsclc

src/base/labels_nebius.config

@@ -173,7 +173,20 @@ process {
     // cannot write Nextflow's root-owned task scratch dir (.command.log /
     // .command.begin / .exitcode -> "Permission denied"). Harmless for already-root
     // GPU images. If segger moves to another gpu label, carry this with it.
-    pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00jp7hyqr094tmy35'], [imagePullPolicy: 'Always'], [runAsUser: 0]]
+    //
+    // emptyDir Memory -> /dev/shm — Kubernetes defaults a container's /dev/shm to 64 MB,
+    // but segger's Lightning training uses a torch_geometric multi-worker DataLoader that
+    // moves each collated graph Batch to the main process through /dev/shm (torch's
+    // file-descriptor sharing strategy, _new_using_fd_cpu). 64 MB overflows on the very
+    // first batch -> "No space left on device (28)" writing /torch_* / "Unexpected bus
+    // error ... insufficient shared memory (shm)", killing the DataLoader workers in the
+    // pre-train sanity-check. NB: the `*sharedmem` labels' `--shm-size` containerOptions do
+    // NOT help here — containerOptions is a docker/singularity concept the k8s executor
+    // ignores; on k8s /dev/shm must be enlarged via a pod volume. The tmpfs only consumes
+    // RAM for pages actually written (idle co-tenants on gpuh100 pay ~0) and counts against
+    // the pod memory limit, so sizeLimit=16Gi sits comfortably under the 120-180 GB request.
+    // Carry this with runAsUser:0 if segger moves labels.
+    pod = [[nodeSelector: 'nebius.com/node-group-id=mk8snodegroup-e00jp7hyqr094tmy35'], [imagePullPolicy: 'Always'], [runAsUser: 0], [emptyDir: [medium: 'Memory', sizeLimit: '16Gi'], mountPath: '/dev/shm']]
     containerOptions = { workflow.containerEngine == "singularity" ? '--nv':
        ( workflow.containerEngine == "docker" ? '--gpus all': null ) }
   }

src/datasets/loaders/bruker_cosmx_nsclc/config.vsh.yaml

@@ -5,9 +5,19 @@ argument_groups:
   - name: Inputs
     arguments:
       - type: string
-        name: --sample
-        example: "Lung9_Rep1"
-        description: "Sample id"
+        name: --input_raw
+        example: "s3://openproblems-data/resources/raw_data/bruker_cosmx/Lung9_Rep1+SMI+Flat+data.tar.gz"
+        description: |
+          URL/path to the flat-files + cell-labels archive (`<sample>+SMI+Flat+data.tar.gz`).
+          Passed as a string (not a staged file) on purpose: the script streams it and extracts
+          only the members sopa needs. Supports s3:// (via s3fs) or a local path.
+      - type: string
+        name: --input_morphology
+        example: "s3://openproblems-data/resources/raw_data/bruker_cosmx/Lung9_Rep1+RawMorphologyImages.tar.gz"
+        description: |
+          URL/path to the raw morphology-images archive (`<sample>+RawMorphologyImages.tar.gz`).
+          Streamed in place; only the 4th z-plane (Z004) of each FOV is extracted. Supports
+          s3:// (via s3fs) or a local path.
       - type: string
         name: --segmentation_id
         default: ["cell"]
@@ -60,9 +70,10 @@ engines:
     __merge__: 
       - /src/base/setup_spatialdata_partial.yaml
     setup:
-      - type: python 
+      - type: python
         pypi:
           - sopa
+          - s3fs  # stream the raw tar.gz archives from s3:// and extract only what sopa needs (see --input_raw)
   - type: native
 
 runners: