Merge remote-tracking branch 'origin/main' into kzm-metabuli-dev

Author: xonq

docs/assets/tables/all_inputs.tsv

@@ -474,9 +474,6 @@ cg_reorder_matrix	disk_size	Int	Amount of storage (in GB) to allocate to the tas
 cg_reorder_matrix	docker	String	The Docker container to use for the task	us-docker.pkg.dev/general-theiagen/staphb/mykrobe:0.12.1	Optional	Snippy_Tree	docker	
 cg_reorder_matrix	memory	Int	Amount of memory/RAM (in GB) to allocate to the task	2	Optional	Snippy_Tree	runtime	
 cg_reorder_matrix	outgroup_root	String	Tip name to root phylogenetic tree upon (overrides midpoint root)		Optional	Snippy_Tree	general	
-cg_snp_dists	cpu	Int	Number of CPUs to allocate to the task	1	Optional	Snippy_Tree	runtime	
-cg_snp_dists	disk_size	Int	Amount of storage (in GB) to allocate to the task	50	Optional	Snippy_Tree	runtime	
-cg_snp_dists	memory	Int	Amount of memory/RAM (in GB) to allocate to the task	2	Optional	Snippy_Tree	runtime	
 checkv_consensus	checkv_db	File	CheckV database file	gs://theiagen-public-resources-rp/reference_data/databases/checkv/checkv-db-v1.5.tar.gz	Optional	TheiaViral_ONT	database	
 checkv_consensus	cpu	Int	Number of CPUs to allocate to the task	2	Optional	TheiaViral_Illumina_PE, TheiaViral_ONT	runtime	
 checkv_consensus	disk_size	Int	Amount of storage (in GB) to allocate to the task	100	Optional	TheiaViral_Illumina_PE, TheiaViral_ONT	runtime	
@@ -628,7 +625,7 @@ core_reorder_matrix	outgroup_root	String	Tip name to root phylogenetic tree upon
 core_snp_dists	cpu	Int	Number of CPUs to allocate to the task	1	Optional	Core_Gene_SNP, kSNP3, kSNP4	runtime	
 core_snp_dists	disk_size	Int	Amount of storage (in GB) to allocate to the task	50	Optional	Core_Gene_SNP, kSNP3, kSNP4	runtime	
 core_snp_dists	docker	String	The Docker container to use for the task	us-docker.pkg.dev/general-theiagen/staphb/snp-dists:0.8.2	Optional	Core_Gene_SNP, kSNP3, kSNP4	docker	
-core_snp_dists	memory	Int	Amount of memory/RAM (in GB) to allocate to the task	2	Optional	Core_Gene_SNP, kSNP3, kSNP4	runtime	
+core_snp_dists	memory	Int	Amount of memory/RAM (in GB) to allocate to the task	4	Optional	Core_Gene_SNP, kSNP3, kSNP4	runtime	
 core_snp_sites	cpu	Int	Number of CPUs to allocate to the task	1	Optional	Core_Gene_SNP	runtime	
 core_snp_sites	disk_size	Int	Amount of storage (in GB) to allocate to the task	100	Optional	Core_Gene_SNP	runtime	
 core_snp_sites	docker	String	Docker image to use for the task	us-docker.pkg.dev/general-theiagen/staphb/snp-sites:2.5.1	Optional	Core_Gene_SNP	docker	
@@ -1851,7 +1848,7 @@ pan_reorder_matrix	outgroup_root	String	Tip name to root phylogenetic tree upon
 pan_snp_dists	cpu	Int	Number of CPUs to allocate to the task	1	Optional	Core_Gene_SNP, kSNP3, kSNP4	runtime	
 pan_snp_dists	disk_size	Int	Amount of storage (in GB) to allocate to the task	50	Optional	Core_Gene_SNP, kSNP3, kSNP4	runtime	
 pan_snp_dists	docker	String	The Docker container to use for the task	us-docker.pkg.dev/general-theiagen/staphb/snp-dists:0.8.2	Optional	Core_Gene_SNP, kSNP3, kSNP4	docker	
-pan_snp_dists	memory	Int	Amount of memory/RAM (in GB) to allocate to the task	2	Optional	Core_Gene_SNP, kSNP3, kSNP4	runtime	
+pan_snp_dists	memory	Int	Amount of memory/RAM (in GB) to allocate to the task	4	Optional	Core_Gene_SNP, kSNP3, kSNP4	runtime	
 pangolin4	analysis_mode	String	Used to switch between usher and pangolearn analysis modes. Only use usher because pangolearn is no longer supported as of Pangolin v4.3 and higher versions.		Optional	Pangolin_Update	general	
 pangolin4	cpu	Int	Number of CPUs to allocate to the task	4	Optional	Pangolin_Update	runtime	
 pangolin4	disk_size	Int	Amount of storage (in GB) to allocate to the task	100	Optional	Pangolin_Update	runtime	
@@ -2268,7 +2265,10 @@ snippy_tree_wf	snippy_core_disk_size	Int	Amount of storage (in GB) to allocate t
 snippy_tree_wf	snippy_core_docker	String	The Docker container to use for the task	us-docker.pkg.dev/general-theiagen/staphb/snippy:4.6.0	Optional	Snippy_Streamline, Snippy_Streamline_FASTA, Snippy_Tree	docker	
 snippy_tree_wf	snippy_core_memory	Int	Amount of memory/RAM (in GB) to allocate to the task	16	Optional	Snippy_Streamline, Snippy_Streamline_FASTA, Snippy_Tree	runtime	
 snippy_tree_wf	snippy_variants_qc_metrics	Array[File]	Files produced by the Snippy_Variants workflow used to concatenate the reports for each sample in the tree		Optional	Snippy_Tree	general	
+snippy_tree_wf	snp_dists_cpu	Int	Number of CPUs to allocate to the task	1	Optional	Snippy_Streamline, Snippy_Streamline_FASTA, Snippy_Tree	runtime	
+snippy_tree_wf	snp_dists_disk_size	Int	Amount of storage (in GB) to allocate to the task	50	Optional	Snippy_Streamline, Snippy_Streamline_FASTA, Snippy_Tree	runtime	
 snippy_tree_wf	snp_dists_docker	String	The Docker container to use for the task	us-docker.pkg.dev/general-theiagen/staphb/snp-dists:0.8.2	Optional	Snippy_Streamline, Snippy_Streamline_FASTA, Snippy_Tree	docker	
+snippy_tree_wf	snp_dists_memory	Int	Amount of memory/RAM (in GB) to allocate to the task	4	Optional	Snippy_Streamline, Snippy_Streamline_FASTA, Snippy_Tree	runtime	
 snippy_tree_wf	snp_sites_cpu	Int	Number of CPUs to allocate to the task	1	Optional	Snippy_Streamline, Snippy_Streamline_FASTA, Snippy_Tree	runtime	
 snippy_tree_wf	snp_sites_disk_size	Int	Amount of storage (in GB) to allocate to the task	100	Optional	Snippy_Streamline, Snippy_Streamline_FASTA, Snippy_Tree	runtime	
 snippy_tree_wf	snp_sites_docker	String	The Docker container to use for the task	us-docker.pkg.dev/general-theiagen/staphb/snp-sites:2.5.1	Optional	Snippy_Streamline, Snippy_Streamline_FASTA, Snippy_Tree	docker	
@@ -2293,7 +2293,7 @@ snippy_variants_wf	read2	File	FASTQ file containing read2 sequences		Optional	Sn
 snp_dists	cpu	Int	Number of CPUs to allocate to the task	1	Optional	Augur	runtime	
 snp_dists	disk_size	Int	Amount of storage (in GB) to allocate to the task	50	Optional	Augur	runtime	
 snp_dists	docker	String	The Docker container to use for the task	us-docker.pkg.dev/general-theiagen/staphb/snp-dists:0.8.2	Optional	Augur	docker	
-snp_dists	memory	Int	Amount of memory/RAM (in GB) to allocate to the task	2	Optional	Augur	runtime	
+snp_dists	memory	Int	Amount of memory/RAM (in GB) to allocate to the task	4	Optional	Augur	runtime	
 sort_bam_assembly_correction	cpu	Int	Number of CPUs to allocate to the task	2	Optional	TheiaMeta_Illumina_PE	runtime	
 sort_bam_assembly_correction	disk_size	Int	Amount of storage (in GB) to allocate to the task	100	Optional	TheiaMeta_Illumina_PE	runtime	
 sort_bam_assembly_correction	docker	String	The Docker container to use for the task	us-docker.pkg.dev/general-theiagen/staphb/samtools:1.17	Optional	TheiaMeta_Illumina_PE	docker	
@@ -2798,9 +2798,6 @@ wg_reorder_matrix	disk_size	Int	Amount of storage (in GB) to allocate to the tas
 wg_reorder_matrix	docker	String	The Docker container to use for the task	us-docker.pkg.dev/general-theiagen/staphb/mykrobe:0.12.1	Optional	Snippy_Tree	docker	
 wg_reorder_matrix	memory	Int	Amount of memory/RAM (in GB) to allocate to the task	2	Optional	Snippy_Tree	runtime	
 wg_reorder_matrix	outgroup_root	String	Tip name to root phylogenetic tree upon (overrides midpoint root)		Optional	Snippy_Tree	general	
-wg_snp_dists	cpu	Int	Number of CPUs to allocate to the task	1	Optional	Snippy_Tree	runtime	
-wg_snp_dists	disk_size	Int	Amount of storage (in GB) to allocate to the task	50	Optional	Snippy_Tree	runtime	
-wg_snp_dists	memory	Int	Amount of memory/RAM (in GB) to allocate to the task	2	Optional	Snippy_Tree	runtime	
 zip_files	cpu	Int	Number of CPUs to allocate to the task	2	Optional	Zip_Column_Content	runtime	
 zip_files	disk_size	Int	Amount of storage (in GB) to allocate to the task	100	Optional	Zip_Column_Content	runtime	
 zip_files	docker_image	String	The Docker container to use for the task	us-docker.pkg.dev/general-theiagen/theiagen/utility:1.1	Optional	Zip_Column_Content	docker	

docs/assets/tables/all_outputs.tsv

@@ -40,6 +40,8 @@ aligned_fai	File	Index file for the reference genome	Clair3_Variants_ONT
 alignment_method	String	The method used to generate the alignment	Freyja_FASTQ
 amr_results_csv	File	CSV formatted AMR profile	AMR_Search
 amr_results_pdf	File	PDF formatted AMR profile	AMR_Search
+amr_search_all_resistances	String	All resistances returned by AMRsearch	AMR_Search, TheiaEuk_Illumina_PE, TheiaEuk_ONT, TheiaProk_FASTA, TheiaProk_Illumina_PE, TheiaProk_Illumina_SE, TheiaProk_ONT
+amr_search_associated_resistances	String	Resistances paired with their agent returned by AMRsearch	AMR_Search, TheiaEuk_Illumina_PE, TheiaEuk_ONT, TheiaProk_FASTA, TheiaProk_Illumina_PE, TheiaProk_Illumina_SE, TheiaProk_ONT
 amr_search_csv	File	CSV formatted AMR profile	TheiaEuk_Illumina_PE, TheiaEuk_ONT, TheiaProk_FASTA, TheiaProk_Illumina_PE, TheiaProk_Illumina_SE, TheiaProk_ONT
 amr_search_docker	String	Docker image used to run AMR_Search	AMR_Search, TheiaEuk_Illumina_PE, TheiaEuk_ONT, TheiaProk_FASTA, TheiaProk_Illumina_PE, TheiaProk_Illumina_SE, TheiaProk_ONT
 amr_search_results	File	JSON formatted AMR profile including BLAST results	AMR_Search, TheiaEuk_Illumina_PE, TheiaEuk_ONT, TheiaProk_FASTA, TheiaProk_Illumina_PE, TheiaProk_Illumina_SE, TheiaProk_ONT
@@ -384,6 +386,7 @@ fastqc_raw2_html	File	An HTML file that provides a graphical visualization of ra
 fastqc_version	String	Version of fastqc software used	Freyja_FASTQ, TheiaCoV_Illumina_PE, TheiaCoV_Illumina_SE, TheiaEuk_Illumina_PE, TheiaMeta_Illumina_PE, TheiaProk_Illumina_PE, TheiaProk_Illumina_SE
 fetch_srr_accession_analysis_date	String	The date the fetch_srr_accession analysis was run.	Fetch_SRR_Accession
 fetch_srr_accession_version	String	The version of the fetch_srr_accession workflow.	Fetch_SRR_Accession
+file_translations	File	A tracking file to use for referencing original filenames and paths when identical files are indexed.	Zip_Column_Content
 filtered_contigs_metrics	File	File containing metrics of contigs filtered	TheiaEuk_Illumina_PE, TheiaEuk_ONT, TheiaProk_Illumina_PE, TheiaProk_Illumina_SE, TheiaProk_ONT
 flu_A_315675_resistance	String	resistance mutations to A_315675	TheiaCoV_FASTA, TheiaCoV_Illumina_PE, TheiaCoV_ONT
 flu_amantadine_resistance	String	resistance mutations to amantadine	TheiaCoV_FASTA, TheiaCoV_Illumina_PE, TheiaCoV_ONT

docs/assets/tables/all_workflows.tsv

@@ -49,4 +49,4 @@ Name	Description	Applicable Kingdom	Workflow Level	Workflow Type	Command-line Co
 [**Transfer_Column_Content**](../workflows/data_export/transfer_column_content.md)	Transfer contents of a specified Terra data table column for many samples ("entities") to a GCP storage bucket location	[Any taxa](../../workflows_overview/workflows_kingdom.md#any-taxa)	Set-level	[Exporting Data from Terra](../../workflows_overview/workflows_type.md#exporting-data-from-terra)	Yes	v1.3.0	[Transfer_Column_Content_PHB](https://dockstore.org/workflows/github.com/theiagen/public_health_bioinformatics/Transfer_Column_Content_PHB:main?tab=info)
 [**Usher**](../workflows/phylogenetic_placement/usher.md)	Use UShER to rapidly and accurately place your samples on any existing phylogenetic tree	Monkeypox virus, SARS-CoV-2, [Viral](../../workflows_overview/workflows_kingdom.md#viral)	Sample-level, Set-level	[Phylogenetic Placement](../../workflows_overview/workflows_type.md#phylogenetic-placement)	Yes	v2.1.0	[Usher_PHB](https://dockstore.org/workflows/github.com/theiagen/public_health_bioinformatics/Usher_PHB:main?tab=info)
 [**VADR_Update**](../workflows/genomic_characterization/vadr_update.md)	Update VADR assignments	HAV, Influenza, Monkeypox virus, RSV-A, RSV-B, SARS-CoV-2, [Viral](../../workflows_overview/workflows_kingdom.md#viral), WNV	Sample-level	[Genomic Characterization](../../workflows_overview/workflows_type.md#genomic-characterization)	Yes	v4.0.0	[VADR_Update_PHB](https://dockstore.org/workflows/github.com/theiagen/public_health_bioinformatics/VADR_Update_PHB:main?tab=info)
-[**Zip_Column_Content**](../workflows/data_export/zip_column_content.md)	Zip contents of a specified Terra data table column for many samples ("entities")	[Any taxa](../../workflows_overview/workflows_kingdom.md#any-taxa)	Set-level	[Exporting Data from Terra](../../workflows_overview/workflows_type.md#exporting-data-from-terra)	Yes	v2.1.0	[Zip_Column_Content_PHB](https://dockstore.org/workflows/github.com/theiagen/public_health_bioinformatics/Zip_Column_Content_PHB:main?tab=info)
+[**Zip_Column_Content**](../workflows/data_export/zip_column_content.md)	Zip contents of a specified Terra data table column for many samples ("entities")	[Any taxa](../../workflows_overview/workflows_kingdom.md#any-taxa)	Set-level	[Exporting Data from Terra](../../workflows_overview/workflows_type.md#exporting-data-from-terra)	Yes	vX.X.X	[Zip_Column_Content_PHB](https://dockstore.org/workflows/github.com/theiagen/public_health_bioinformatics/Zip_Column_Content_PHB:main?tab=info)

docs/workflows/data_export/zip_column_content.md

@@ -10,6 +10,8 @@ title: Zip_Column_Content
 
 This workflow will create a zip file containing all of the items from a given column in a Terra Data Table. This is useful when you want to share a collection of result files.
 
+If a column contains files that do not have unique filenames then an index will be appended to identical filenames. Original file paths and names are preserved and referenced within the `file_translations.tsv` output. 
+
 ### Inputs
 
 This workflow runs on the _set_ level.

tasks/gene_typing/drug_resistance/task_amr_search.wdl

@@ -28,12 +28,30 @@ task amr_search {
     python3 /scripts/parse_amr_json.py \
         ./~{samplename}_paarsnp_results.jsn \
         ~{samplename}
+    
+    # Fix carriage return characters
+    sed -i 's/\r$//' "~{samplename}_amr_results.csv"
+
+    # Pull all resistances and place them into comma separated string similar to AMRFinder
+    awk -F ',' '/Resistant/ {gsub(/; /, "\n", $3); print $3}' "~{samplename}_amr_results.csv" | sort -u | paste -sd ',' -  > RESISTANCES
+
+    # Paired resistances with agent
+    # Place into Agent_1: gene/mutation, gene/mutation; Agent_2: gene/mutation, gene/mutation; format
+    awk -F ',' '/Resistant/ {gsub("; ", ", ", $3); printf "%s: %s; ", $1, $3}' "~{samplename}_amr_results.csv" | sed 's/; $//' > ASSOCIATED_RESISTANCES
+
+    if [[ ! -s RESISTANCES || "$(cat RESISTANCES)" == "none" ]]; then
+      echo "No resistances reported" > RESISTANCES
+      echo "No resistances reported" > ASSOCIATED_RESISTANCES
+    fi
+
   >>>
   output {
     File amr_search_json_output = "~{samplename}_paarsnp_results.jsn"
     File amr_search_output_csv = "~{samplename}_amr_results.csv"
     File amr_search_output_pdf = "~{samplename}_amr_results.pdf"
     String amr_search_version = read_string("output_amr_version.txt")
+    String amr_search_all_resistances = read_string("RESISTANCES")
+    String amr_search_associated_resistances = read_string("ASSOCIATED_RESISTANCES")
     String amr_search_docker_image = docker
   }
 

tasks/phylogenetic_inference/utilities/task_snp_dists.wdl

@@ -6,7 +6,7 @@ task snp_dists {
     String cluster_name
     Int disk_size = 50
     String docker = "us-docker.pkg.dev/general-theiagen/staphb/snp-dists:0.8.2"
-    Int memory = 2
+    Int memory = 4
     Int cpu = 1
   }
   command <<<

tasks/utilities/file_handling/task_zip_files.wdl

@@ -16,17 +16,45 @@ task zip_files {
   command <<<
     file_array=(~{sep=' ' files_to_zip})
     mkdir ~{zipped_file_name}
+    echo -e "origin_path\tnew_path" > file_translations.tsv
 
-    # move files oto a single directory before zipping
-    for index in ${!file_array[@]}; do
-      file=${file_array[$index]}
-      mv ${file} ~{zipped_file_name}
+    # move files into a single directory before zipping
+    for file in "${file_array[@]}"; do
+    
+      echo "DEBUG: Pulling $file"
+      if [ -f "$file" ]; then
+        echo "DEBUG: $file exists"
+        filename=$(basename "$file") # Extract the filename (e.g., test.tsv)
+        dest="~{zipped_file_name}/$filename"
+
+        # Counter is always set to 1 so that if there are 
+        # other duplicated filenames they will be counted as well.
+        counter=1
+
+        echo "DEBUG: Checking for $file in $dest"
+        # Check for duplicate files in the destination
+        while [ -e "$dest" ]; do 
+          echo "DEBUG: Duplicate filename found, adding a file index for differentiation."
+          dest="~{zipped_file_name}/${filename%.*}_${counter}.${filename##*.}"
+          echo "DEBUG: New filename ${filename%.*}_${counter}.${filename##*.}"
+          ((counter++))
+        done
+
+        # Move the file to the destination with the new name
+        # If loop is not entered, filename will remain unchanged. 
+        mv "$file" "$dest"
+        echo -e "$file\t$dest" >> file_translations.tsv
+
+      else
+        echo "File not found: $file"
+      fi
     done
     
     zip -r ~{zipped_file_name}.zip ~{zipped_file_name}
    >>>
   output {
     File zipped_files = "~{zipped_file_name}.zip"
+    File file_translations = "file_translations.tsv"
   }
   runtime {
     docker: "~{docker_image}"

tests/workflows/theiaprok/test_wf_theiaprok_illumina_pe.yml

@@ -456,7 +456,7 @@
     - path: miniwdl_run/wdl/tasks/taxon_id/contamination/task_midas.wdl
       md5sum: 481f5fbce5aa1abf93acd912797821cc
     - path: miniwdl_run/wdl/workflows/theiaprok/wf_theiaprok_illumina_pe.wdl
-      md5sum: 68ca5ef61b5b66fcae09ce03b4d7de1a
+      md5sum: e5626b062172e8bff267aa04cdedf5e9
     - path: miniwdl_run/wdl/workflows/utilities/wf_read_QC_trim_pe.wdl
       contains: ["version", "QC", "output"]
     - path: miniwdl_run/workflow.log

tests/workflows/theiaprok/test_wf_theiaprok_illumina_se.yml

@@ -436,7 +436,7 @@
     - path: miniwdl_run/wdl/tasks/taxon_id/contamination/task_midas.wdl
       md5sum: 481f5fbce5aa1abf93acd912797821cc
     - path: miniwdl_run/wdl/workflows/theiaprok/wf_theiaprok_illumina_se.wdl
-      md5sum: 65d2d61e8a83a5b2b37ae50773ab04af
+      md5sum: 815c06e0ff6c90c172be481684795dda
     - path: miniwdl_run/workflow.log
       contains: ["wdl", "theiaprok_illumina_se", "NOTICE", "done"]
 

workflows/phylogenetics/wf_snippy_tree.wdl

@@ -51,6 +51,9 @@ workflow snippy_tree_wf {
     Int? iqtree2_ultrafast_bootstraps
     String? iqtree2_model
 
+    Int? snp_dists_cpu
+    Int? snp_dists_memory
+    Int? snp_dists_disk_size
     String? snp_dists_docker
 
     Int? snp_sites_cpu
@@ -130,7 +133,10 @@ workflow snippy_tree_wf {
     input:
       alignment = select_first([gubbins.gubbins_polymorphic_fasta, snippy_core.snippy_full_alignment_clean]),
       cluster_name = tree_name_updated,
-      docker = snp_dists_docker
+      docker = snp_dists_docker,
+      cpu = snp_dists_cpu,
+      memory = snp_dists_memory,
+      disk_size = snp_dists_disk_size
   }
   # mid-point roots the phylogenetic tree, and reorders the columns in the wgSNP matrix according to the tree tip order
   # NB the tree will remain a core genome tree is core_genome = true, and a whole-genome tree if core_genome = false
@@ -148,7 +154,10 @@ workflow snippy_tree_wf {
       input:
         alignment = select_first([snp_sites.snp_sites_multifasta]),
         cluster_name = tree_name_updated,
-        docker = snp_dists_docker
+        docker = snp_dists_docker,   
+        cpu = snp_dists_cpu,
+        memory = snp_dists_memory,
+        disk_size = snp_dists_disk_size
     }
     # reorders the columns in the cgSNP matrix according to the tree tip order
     # input tree is the midpoint rooted tree from the wg_reorder_matrix task, and midpoint rooting is turned off here, so the tree remains unchanged