Apply fixes for scripts

CHANGELOG.md

@@ -1,3 +1,7 @@
 # datasets v0.1.0
 
-* Port dataset loaders and processors from openproblems-bio/openproblems
+* Port dataset loaders and processors from `openproblems-bio/openproblems`.
+
+* Port OP3 data loader (PR #2).
+
+* Apply fixes to scripts and components (PR #3).

scripts/create_resources/cellxgene_census.sh

@@ -39,7 +39,7 @@ param_list:
     dataset_url: https://cellxgene.cziscience.com/collections/296237e2-393d-4e31-b590-b03f74ac5070
     dataset_reference: hrovatin2023delineating
     dataset_organism: mus_musculus
-  - id: cellxgene_census/hcla
+  - id: cellxgene_census/hlca
     species: homo_sapiens
     census_version: "2023-07-25"
     obs_value_filter: "dataset_id == '066943a2-fdac-4b29-b348-40cede398e4e'"
@@ -142,10 +142,10 @@ process {
 }
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems.git \
-  --revision main_build \
+tw launch https://github.com/openproblems-bio/datasets.git \
+  --revision build/main \
   --pull-latest \
-  --main-script target/nextflow/datasets/workflows/scrnaseq/process_cellxgene_census/main.nf \
+  --main-script target/nextflow/workflows/scrnaseq/process_cellxgene_census/main.nf \
   --workspace 53907369739130 \
   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
   --params-file "/tmp/params.yaml" \

scripts/create_resources/dataset_info.sh

@@ -31,11 +31,11 @@ process {
 }
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems.git \
-  --revision main_build \
+tw launch https://github.com/openproblems-bio/datasets.git \
+  --revision build/main \
   --entry-name auto \
   --pull-latest \
-  --main-script target/nextflow/datasets/workflows/extract_dataset_info/main.nf \
+  --main-script target/nextflow/workflows/extract_dataset_info/main.nf \
   --workspace 53907369739130 \
   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
   --params-file "/tmp/params.yaml" \

scripts/create_resources/openproblems_neurips2021_multimodal.sh

@@ -35,10 +35,10 @@ output_state: '$id/state.yaml'
 publish_dir: s3://openproblems-data/resources/datasets/multimodal
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems.git \
-  --revision main_build \
+tw launch https://github.com/openproblems-bio/datasets.git \
+  --revision build/main \
   --pull-latest \
-  --main-script target/nextflow/datasets/workflows/multimodal/process_openproblems_neurips2021_bmmc/main.nf \
+  --main-script target/nextflow/workflows/multimodal/process_openproblems_neurips2021_bmmc/main.nf \
   --workspace 53907369739130 \
   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
   --params-file "$params_file" \

scripts/create_resources/openproblems_neurips2021_multimodal_test.sh

@@ -37,7 +37,7 @@ HERE
 
 export NXF_VER=23.10.1
 nextflow run . \
-  -main-script target/nextflow/datasets/workflows/multimodal/process_openproblems_neurips2021_bmmc/main.nf \
+  -main-script target/nextflow/workflows/multimodal/process_openproblems_neurips2021_bmmc/main.nf \
   -profile docker \
   -resume \
   -params-file "$params_file"

scripts/create_resources/openproblems_neurips2022_pbmc.sh

@@ -46,10 +46,10 @@ process {
 }
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems.git \
-  --revision main_build \
+tw launch https://github.com/openproblems-bio/datasets.git \
+  --revision build/main \
   --pull-latest \
-  --main-script target/nextflow/datasets/workflows/multimodal/process_openproblems_neurips2022_pbmc/main.nf \
+  --main-script target/nextflow/workflows/multimodal/process_openproblems_neurips2022_pbmc/main.nf \
   --workspace 53907369739130 \
   --compute-env 1pK56PjjzeraOOC2LDZvN2 \
   --params-file "$params_file" \

scripts/create_resources/openproblems_op3.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#It is an example how to set parameters and execute processing for the op3 dataset.
+set -e
+
+params_file="/tmp/datasets_op3.yaml"
+
+cat > "$params_file" << 'HERE'
+param_list:
+  - id: openproblems_op3
+    input: https://ftp.ncbi.nlm.nih.gov/geo/series/GSE279nnn/GSE279945/suppl/GSE279945_sc_counts_processed.h5ad
+    dataset_name: "OP3: single-cell multimodal dataset in PBMCs for perturbation prediction benchmarking"
+    dataset_summary: "The Open Problems Perurbation Prediction (OP3) dataset with small molecule perturbations in PBMCs"
+    dataset_description: "The OP3 dataset is to-date the largest single-cell small molecule perturbation dataset in primary tissue with multiple donor replicates."
+    dataset_url: "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE279nnn/GSE279945/suppl/GSE279945_sc_counts_processed.h5ad"
+    dataset_reference: GSE279945
+normalization_methods: 
+  - log_cp10k
+output_dataset: '$id/dataset.h5ad'
+do_subsample: False
+output_meta: '$id/dataset_metadata.yaml'
+output_state: '$id/state.yaml'
+output_raw: force_null
+output_normalized: force_null
+output_pca: force_null
+output_hvg: force_null
+output_knn: force_null
+publish_dir: s3://openproblems-data/resources/datasets
+HERE
+
+cat > "/tmp/nextflow.config" << 'HERE'
+process {
+  withName:'.*publishStatesProc' {
+    memory = '100GB'
+    disk = '1000GB'
+  }
+}
+HERE
+
+tw launch https://github.com/openproblems-bio/datasets.git \
+ --revision build/main \
+ --pull-latest \
+ --main-script target/nextflow/workflows/scrnaseq/process_openproblems_op3/main.nf \
+ --workspace 53907369739130 \
+ --params-file "$params_file" \
+ --labels op3,dataset_loader \
+ --config /tmp/nextflow.config
+

scripts/create_resources/openproblems_op3_test.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+cd "$REPO_ROOT"
+
+DATASET_DIR=op3_data/
+
+set -e
+mkdir -p $DATASET_DIR
+
+cat > "/tmp/nextflow.config" << 'HERE'
+process {
+  withName:'.*publishStatesProc' {
+    memory = '100GB'
+    disk = '1000GB'
+  }
+}
+HERE
+
+set -x
+nextflow run . \
+  -main-script target/nextflow/workflows/scrnaseq/process_openproblems_op3/main.nf \
+  -profile docker \
+  -resume \
+  --input https://ftp.ncbi.nlm.nih.gov/geo/series/GSE279nnn/GSE279945/suppl/GSE279945_sc_counts_processed.h5ad \
+  --id openproblems_sop3 \
+  --dataset_name "OP3: single-cell multimodal dataset in PBMCs for perturbation prediction benchmarking" \
+  --dataset_summary "The Open Problems Perurbation Prediction (OP3) dataset with small molecule perturbations in PBMCs" \
+  --dataset_description "The OP3 dataset is to-date the largest single-cell small molecule perturbation dataset in primary tissue with multiple donor replicates." \
+  --dataset_url "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE279nnn/GSE279945/suppl/GSE279945_sc_counts_processed.h5ad" \
+  --dataset_reference GSE279945 \
+  --do_subsample False \
+  --normalization_methods log_cp10k \
+  --output_dataset '$id/dataset.h5ad' \
+  --output_meta '$id/dataset_metadata.yaml' \
+  --output_state '$id/state.yaml' \
+  --output_raw force_null \
+  --output_normalized force_null \
+  --output_pca force_null \
+  --output_hvg force_null \
+  --output_knn force_null \
+  --publish_dir $DATASET_DIR \
+  -config /tmp/nextflow.config

scripts/create_resources/openproblems_v1.sh

@@ -171,10 +171,10 @@ process {
 }
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems.git \
-  --revision main_build \
+tw launch https://github.com/openproblems-bio/datasets.git \
+  --revision build/main \
   --pull-latest \
-  --main-script target/nextflow/datasets/workflows/scrnaseq/process_openproblems_v1/main.nf \
+  --main-script target/nextflow/workflows/scrnaseq/process_openproblems_v1/main.nf \
   --workspace 53907369739130 \
   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
   --params-file "$params_file" \

scripts/create_resources/openproblems_v1_multimodal.sh

@@ -74,10 +74,10 @@ process {
 }
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems.git \
-  --revision main_build \
+tw launch https://github.com/openproblems-bio/datasets.git \
+  --revision build/main \
   --pull-latest \
-  --main-script target/nextflow/datasets/workflows/multimodal/process_openproblems_v1_multimodal/main.nf \
+  --main-script target/nextflow/workflows/multimodal/process_openproblems_v1_multimodal/main.nf \
   --workspace 53907369739130 \
   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
   --params-file "$params_file" \

scripts/create_resources/openproblems_v1_multimodal_test.sh

@@ -38,7 +38,7 @@ HERE
 export NXF_VER=22.04.5
 nextflow \
   run . \
-  -main-script target/nextflow/datasets/workflows/multimodal/process_openproblems_v1_multimodal/main.nf \
+  -main-script target/nextflow/workflows/multimodal/process_openproblems_v1_multimodal/main.nf \
   -profile docker \
   -resume \
   -params-file "$params_file" \

scripts/create_resources/openproblems_v1_test.sh

@@ -42,7 +42,7 @@ HERE
 
 export NXF_VER=23.04.2
 nextflow run . \
-  -main-script target/nextflow/datasets/workflows/scrnaseq/process_openproblems_v1/main.nf \
+  -main-script target/nextflow/workflows/scrnaseq/process_openproblems_v1/main.nf \
   -profile docker \
   -resume \
   -params-file "$params_file" \

scripts/create_resources/tenx_visium.sh

@@ -293,10 +293,10 @@
 # publish_dir: s3://openproblems-data/resources/datasets/spatial
 # HERE
 
-tw launch https://github.com/openproblems-bio/openproblems.git \
-  --revision main_build \
+tw launch https://github.com/openproblems-bio/datasets.git \
+  --revision build/main \
   --pull-latest \
-  --main-script target/nextflow/datasets/workflows/spatial/process_tenx_visium/main.nf \
+  --main-script target/nextflow/workflows/spatial/process_tenx_visium/main.nf \
   --workspace 53907369739130 \
   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
   --params-file "/tmp/params.yaml" \

scripts/create_resources/zenodo.sh

@@ -404,10 +404,10 @@ process {
 }
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems.git \
-  --revision main_build \
+tw launch https://github.com/openproblems-bio/datasets.git \
+  --revision build/main \
   --pull-latest \
-  --main-script target/nextflow/datasets/workflows/spatial/process_zenodo/main.nf \
+  --main-script target/nextflow/workflows/spatial/process_zenodo/main.nf \
   --workspace 53907369739130 \
   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
   --params-file "/tmp/params.yaml" \

scripts/create_resources/zenodo_slidetags.sh

@@ -72,10 +72,10 @@ process {
 }
 HERE
 
-tw launch https://github.com/openproblems-bio/openproblems.git \
-  --revision main_build \
+tw launch https://github.com/openproblems-bio/datasets.git \
+  --revision build/main \
   --pull-latest \
-  --main-script target/nextflow/datasets/workflows/spatial/process_zenodo_slidetags/main.nf \
+  --main-script target/nextflow/workflows/spatial/process_zenodo_slidetags/main.nf \
   --workspace 53907369739130 \
   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
   --params-file "/tmp/params.yaml" \

scripts/create_test_resources/cxg_immune_cell_atlas.sh

@@ -39,7 +39,7 @@ keep_features: '$KEEP_FEATURES'
 HERE
 
 nextflow run . \
-  -main-script target/nextflow/datasets/workflows/scrnaseq/process_cellxgene_census/main.nf \
+  -main-script target/nextflow/workflows/scrnaseq/process_cellxgene_census/main.nf \
   -c src/wf_utils/labels_ci.config \
   -profile docker \
   -params-file "/tmp/params.yaml"

scripts/create_test_resources/cxg_mouse_pancreas_atlas.sh

@@ -40,7 +40,7 @@ keep_features: '$KEEP_FEATURES'
 HERE
 
 nextflow run . \
-  -main-script target/nextflow/datasets/workflows/scrnaseq/process_cellxgene_census/main.nf \
+  -main-script target/nextflow/workflows/scrnaseq/process_cellxgene_census/main.nf \
   -c src/wf_utils/labels_ci.config \
   -profile docker \
   -params-file "/tmp/params.yaml"

scripts/create_test_resources/mouse_brain_coronal.sh

@@ -30,7 +30,7 @@ remove_mitochondrial: true
 HERE
 
 nextflow run . \
-  -main-script target/nextflow/datasets/workflows/spatial/process_tenx_visium/main.nf \
+  -main-script target/nextflow/workflows/spatial/process_tenx_visium/main.nf \
   -c src/wf_utils/labels_ci.config \
   -profile docker \
   -params-file "/tmp/params.yaml"

scripts/create_test_resources/neurips2021_bmmc.sh

@@ -51,16 +51,16 @@ HERE
 # HERE
 
 nextflow run . \
-  -main-script target/nextflow/datasets/workflows/multimodal/process_openproblems_neurips2021_bmmc/main.nf \
+  -main-script target/nextflow/workflows/multimodal/process_openproblems_neurips2021_bmmc/main.nf \
   -profile docker \
   -resume \
   --publish_dir resources_test/common/multimodal \
   -params-file "$params_file" \
   -c src/wf_utils/labels.config
 
-# tw launch https://github.com/openproblems-bio/openproblems.git \
-#   --revision main_build \
-#   --main-script target/nextflow/datasets/workflows/process_openproblems_neurips2021_bmmc/main.nf \
+# tw launch https://github.com/openproblems-bio/datasets.git \
+#   --revision build/main \
+#   --main-script target/nextflow/workflows/process_openproblems_neurips2021_bmmc/main.nf \
 #   --workspace 53907369739130 \
 #   --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
 #   --params-file "$params_file" \

scripts/create_test_resources/neurips2022_pbmc.sh

@@ -42,7 +42,7 @@ publish_dir: s3://openproblems-data/resources_test/common/multimodal
 HERE
 
 # nextflow run . \
-#   -main-script target/nextflow/datasets/workflows/multimodal/process_openproblems_neurips2022_pbmc/main.nf \
+#   -main-script target/nextflow/workflows/multimodal/process_openproblems_neurips2022_pbmc/main.nf \
 #   -profile docker \
 #   -resume \
 #   --publish_dir resources_test/common \
@@ -60,10 +60,10 @@ process {
 HERE
 
 
-tw launch https://github.com/openproblems-bio/openproblems.git \
-  --revision main_build \
+tw launch https://github.com/openproblems-bio/datasets.git \
+  --revision build/main \
   --pull-latest \
-  --main-script target/nextflow/datasets/workflows/multimodal/process_openproblems_neurips2022_pbmc/main.nf \
+  --main-script target/nextflow/workflows/multimodal/process_openproblems_neurips2022_pbmc/main.nf \
   --workspace 53907369739130 \
   --compute-env 1pK56PjjzeraOOC2LDZvN2 \
   --params-file "$params_file" \

scripts/create_test_resources/pancreas.sh

@@ -18,7 +18,7 @@ KEEP_FEATURES=`cat $DATASET_DIR/temp_g2m_genes_tirosh_hm.txt $DATASET_DIR/temp_s
 
 # download dataset
 nextflow run . \
-  -main-script target/nextflow/datasets/workflows/scrnaseq/process_openproblems_v1/main.nf \
+  -main-script target/nextflow/workflows/scrnaseq/process_openproblems_v1/main.nf \
   -profile docker \
   -c src/wf_utils/labels_ci.config \
   -resume \

scripts/create_test_resources/scicar_cell_lines.sh

@@ -14,7 +14,7 @@ mkdir -p $DATASET_DIR
 
 # download dataset
 nextflow run . \
-  -main-script target/nextflow/datasets/workflows/multimodal/process_openproblems_v1_multimodal/main.nf \
+  -main-script target/nextflow/workflows/multimodal/process_openproblems_v1_multimodal/main.nf \
   -profile docker \
   -resume \
   --id scicar_cell_lines \

src/api/comp_dataset_loader.yaml

@@ -1,4 +1,3 @@
-# namespace: loaders
 info:
   type: dataset_loader
   type_info:
@@ -7,9 +6,16 @@ info:
     description: |
       A dataset loader will typically have an identifier (e.g. a GEO identifier)
       or URL as input argument and additional arguments to define where the script needs to download a dataset from and how to process it.
-arguments:
-  - name: "--output"
-    __merge__: file_raw.yaml
-    direction: "output"
-    required: true
-test_resources: []
+argument_groups:
+  - name: Output
+    description: "Output parameters"
+    arguments:
+      - name: "--output"
+        __merge__: file_raw.yaml
+        direction: output
+        required: true
+      - name: "--output_compression"
+        type: string
+        choices: [gzip, lzf]
+        required: false
+        default: "gzip"

src/api/comp_normalization.yaml

@@ -16,6 +16,11 @@ arguments:
     __merge__: file_normalized.yaml
     direction: output
     required: true
+  - name: "--output_compression"
+    type: string
+    choices: [gzip, lzf]
+    required: false
+    default: "gzip"
   - name: "--normalization_id"
     type: string
     description: "The normalization id to store in the dataset metadata. If not specified, the functionality name will be used."