From 30bfb4b43eee7c8b4e791fec994f23342b4a657a Mon Sep 17 00:00:00 2001 From: Aris Paschalidis Date: Fri, 29 Oct 2021 16:55:31 -0400 Subject: [PATCH 01/11] Install basespace CLI --- MIPTools.def | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MIPTools.def b/MIPTools.def index d3810f3..f594d1f 100644 --- a/MIPTools.def +++ b/MIPTools.def @@ -186,6 +186,10 @@ From: amd64/ubuntu:20.04 # install parasight scp /opt/programs/parasight_v7.6/parasight.pl /opt/bin/parasight76.pl + # install basespace cli + BS_PATH="https://launch.basespace.illumina.com/CLI/latest/amd64-linux/bs" + wget $BS_PATH -O /opt/bin/bs + # add executable flag to executables chmod -R +xr /usr/bin chmod -R +xr /opt/bin From 968358062c64f4e9984bfd87a1c5957739fe013d Mon Sep 17 00:00:00 2001 From: Aris Paschalidis Date: Mon, 1 Nov 2021 18:33:51 -0400 Subject: [PATCH 02/11] Remove old python download script --- bin/BaseSpaceRunDownloader_v2.py | 91 -------------------------------- 1 file changed, 91 deletions(-) delete mode 100644 bin/BaseSpaceRunDownloader_v2.py diff --git a/bin/BaseSpaceRunDownloader_v2.py b/bin/BaseSpaceRunDownloader_v2.py deleted file mode 100644 index 9709a53..0000000 --- a/bin/BaseSpaceRunDownloader_v2.py +++ /dev/null @@ -1,91 +0,0 @@ -from urllib.request import Request, urlopen, URLError -import json -import math -import sys -import os -import socket -import optparse - - -def arg_parser(): - parser = optparse.OptionParser() - parser.add_option('-r', dest='runid', help='Run ID: required') - parser.add_option('-a', dest='accesstoken', help='Access Token: required') - (options, args) = parser.parse_args() - try: - if options.runid is None: - raise Exception - if options.accesstoken is None: - raise Exception - except Exception: - print("Usage: BaseSpaceRunDownloader_vN.py -r " - " -a ") - sys.exit() - return options - - -def restrequest(rawrequest): - request = Request(rawrequest) - try: - response = urlopen(request) - json_string = response.read() - json_obj = json.loads(json_string) - except URLError as e: - print('Got an error code:', e) - sys.exit() - return json_obj - - -def downloadrestrequest(rawrequest, path): - dirname = RunID + os.sep + os.path.dirname(path) - if dirname != '': - if not os.path.isdir(dirname): - os.makedirs(dirname) - request = (rawrequest) - outfile = open(RunID + os.sep + path, 'wb') - try: - response = urlopen(request, timeout=1) - outfile.write(response.read()) - outfile.close() - except URLError as e: - print('Got an error code:', e) - outfile.close() - downloadrestrequest(rawrequest, path) - except socket.error: - print('Got a socket error: retrying') - outfile.close() - downloadrestrequest(rawrequest, path) - - -options = arg_parser() -RunID = options.runid -AccessToken = options.accesstoken -request = ('http://api.basespace.illumina.com/v1pre3/runs/{}/files?' - 'access_token={}').format(RunID, AccessToken) -json_obj = restrequest(request) -totalCount = json_obj['Response']['TotalCount'] -noffsets = int(math.ceil(float(totalCount)/1000.0)) -hreflist = [] -pathlist = [] -filenamelist = [] - -for index in range(noffsets): - offset = 1000*index - request = ( - 'http://api.basespace.illumina.com/v1pre3/runs/{}/files' - '?access_token={}&limit=1000&Offset={}' - ).format(RunID, AccessToken, offset) - json_obj = restrequest(request) - nfiles = len(json_obj['Response']['Items']) - for fileindex in range(nfiles): - href = json_obj['Response']['Items'][fileindex]['Href'] - hreflist.append(href) - path = json_obj['Response']['Items'][fileindex]['Path'] - pathlist.append(path) - -for index in range(len(hreflist)): - request = ( - 'http://api.basespace.illumina.com/{}/content?access_token={}' - ).format(hreflist[index], AccessToken) - print('downloading {}'.format(pathlist[index])) - downloadrestrequest(request, pathlist[index]) From 429139371ff67dc60d357c8a79bb3ba9783546b2 Mon Sep 17 00:00:00 2001 From: Aris Paschalidis Date: Mon, 1 Nov 2021 18:36:19 -0400 Subject: [PATCH 03/11] Define basespace download app --- MIPTools.def | 53 +++++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/MIPTools.def b/MIPTools.def index f594d1f..e5134e3 100644 --- a/MIPTools.def +++ b/MIPTools.def @@ -338,16 +338,15 @@ From: amd64/ubuntu:20.04 . /opt/analysis/wrangle.sh ################################################################## -## Download App ## +## Basespace Download App ## ################################################################## -%apprun download - # Parse options +%apprun bs_download set -e - set -u - while getopts r: opt; do - case $opt in - r) run_id=$OPTARG;; - ?) echo "Usage: singularity run --app download \\" + + # Set default values for paths + output_path="/opt/analysis" + config_path="/opt/resources/basespace.cfg" + echo " -B /path_to_output_dir:/opt/analysis \\" echo " -B /path_to_base_resources:/opt/resources \\" echo " mycontainer.sif -r my_Illumina_run_ID" @@ -357,23 +356,31 @@ From: amd64/ubuntu:20.04 echo "must be mounted to /opt/data." exit 1;; esac + + # Argument handling using getopts + # Could alternatively use manual processing. This would allow for longer + # formed inputs. + while getopts 'i:o:c:' opt; do + case "${opt}" in + i) run_id=${OPTARG} ;; + o) ouput_path=${OPTARG} ;; + c) config_path="${OPTARG}" ;; + esac done - - # Print to CLI - echo "Downloading NextSeq run $run_id from BaseSpace." - echo "Depending on the data size, this can take very long (up to 10 h)" - echo "It is recommended to run this app in a screen (GNU screen)." - echo "A message indicating the end of download will be printed when done." - echo "Check nohup.out file in your output directory for the download log." - # cd and run app - # Use nohup to make command keep running even if get hangup signal - cd /opt/analysis - nohup python /opt/bin/BaseSpaceRunDownloader_v2.py \ - -r $run_id -a "$(cat /opt/resources/access_token.txt)" - - # Print to CLI - echo "Download finished." + # Ensure run_id is specified + if [ ! "$run_id" ]; then + echo "Argument -i must be provided" + echo "$usage" >&2; exit 1 + fi + + # Read data from config file + # Remove whitespace from each line and export each line as a variable + export BASESPACE_API_SERVER=$(sed '1q;d' ${config_path} | sed 's/.*=.//g') + export BASESPACE_ACCESS_TOKEN=$(sed '2q;d' ${config_path} | sed 's/.*=.//g') + + # Download data + bs download run -i ${run_id} -o ${output_path} ################################################################# ## Demux App ## From cf7e317603b4a879d3289b1f35bdefe105cb2a5b Mon Sep 17 00:00:00 2001 From: Aris Paschalidis Date: Mon, 1 Nov 2021 18:37:37 -0400 Subject: [PATCH 04/11] Add help page --- MIPTools.def | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/MIPTools.def b/MIPTools.def index e5134e3..999ec84 100644 --- a/MIPTools.def +++ b/MIPTools.def @@ -347,15 +347,26 @@ From: amd64/ubuntu:20.04 output_path="/opt/analysis" config_path="/opt/resources/basespace.cfg" - echo " -B /path_to_output_dir:/opt/analysis \\" - echo " -B /path_to_base_resources:/opt/resources \\" - echo " mycontainer.sif -r my_Illumina_run_ID" - echo "An 'access_token.txt' file with a valid access token is " - echo "required. It must be present in base_resources directory." - echo "A data directory where the data will be downloaded to" - echo "must be mounted to /opt/data." - exit 1;; - esac + help() { + echo "Download data from the Illumina BaseSpace Sequence Hub" + echo "" + echo "Usage:" + echo " singularity run [options] --app bs_download "\ + "[app_options]" + echo "" + echo "Options:" + echo " See 'singularity run'" + echo "" + echo "App Options:" + echo " -i Required. The run ID of the data to download." + echo " -o The path to the output directory." + echo " Default: '/opt/analysis'." + echo " -c The path to the authentication credentials file." + echo " This file is created by 'bs auth'. For additional" + echo " information see the help page for that command." + echo " Default: '/opt/resources/basespace.cfg'." + echo " -h Print the help page." + } # Argument handling using getopts # Could alternatively use manual processing. This would allow for longer @@ -365,6 +376,10 @@ From: amd64/ubuntu:20.04 i) run_id=${OPTARG} ;; o) ouput_path=${OPTARG} ;; c) config_path="${OPTARG}" ;; + h) help + exit 1 ;; + *) help + exit 1 ;; esac done From b67ffafa93afe1b7d3be55fe8b683815b41b9e23 Mon Sep 17 00:00:00 2001 From: Aris Paschalidis Date: Mon, 1 Nov 2021 18:39:30 -0400 Subject: [PATCH 05/11] Add template basespace authentication file --- base_resources/basespace.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 base_resources/basespace.cfg diff --git a/base_resources/basespace.cfg b/base_resources/basespace.cfg new file mode 100644 index 0000000..34fa528 --- /dev/null +++ b/base_resources/basespace.cfg @@ -0,0 +1,2 @@ +apiServer = https://api.basespace.illumina.com +accessToken = From d87cd34bdfe7002e837ca3dc89c89ca92bf6867e Mon Sep 17 00:00:00 2001 From: Aris Paschalidis Date: Mon, 1 Nov 2021 18:41:31 -0400 Subject: [PATCH 06/11] Add -h as a flag --- MIPTools.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MIPTools.def b/MIPTools.def index 999ec84..9eb2e7f 100644 --- a/MIPTools.def +++ b/MIPTools.def @@ -371,7 +371,7 @@ From: amd64/ubuntu:20.04 # Argument handling using getopts # Could alternatively use manual processing. This would allow for longer # formed inputs. - while getopts 'i:o:c:' opt; do + while getopts 'i:o:c:h' opt; do case "${opt}" in i) run_id=${OPTARG} ;; o) ouput_path=${OPTARG} ;; From 2fa35edabf7d4bbb4b01847a1be52c6211529504 Mon Sep 17 00:00:00 2001 From: Aris Paschalidis Date: Mon, 1 Nov 2021 19:07:50 -0400 Subject: [PATCH 07/11] Add an example to help page --- MIPTools.def | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/MIPTools.def b/MIPTools.def index 9eb2e7f..9b32948 100644 --- a/MIPTools.def +++ b/MIPTools.def @@ -348,14 +348,14 @@ From: amd64/ubuntu:20.04 config_path="/opt/resources/basespace.cfg" help() { - echo "Download data from the Illumina BaseSpace Sequence Hub" + echo "Download data from the Illumina BaseSpace Sequence Hub." echo "" echo "Usage:" echo " singularity run [options] --app bs_download "\ "[app_options]" echo "" echo "Options:" - echo " See 'singularity run'" + echo " See 'singularity run'." echo "" echo "App Options:" echo " -i Required. The run ID of the data to download." @@ -366,11 +366,22 @@ From: amd64/ubuntu:20.04 echo " information see the help page for that command." echo " Default: '/opt/resources/basespace.cfg'." echo " -h Print the help page." + echo "" + echo "Examples:" + echo " # Set paths" + echo " $ resource_dir=/bin/MIPTools/base_resources" + echo " $ run_dir=/work/usr/example" + echo "" + echo " # Run app" + echo " $ singularity run \\" + echo " -B \$resource_dir:/opt/resources"\ + "-B \$run_dir:/opt/analysis \\" + echo " --app bs_download -i " } # Argument handling using getopts - # Could alternatively use manual processing. This would allow for longer - # formed inputs. + # Could alternatively use manual processing. This would allow for long + # form inputs. while getopts 'i:o:c:h' opt; do case "${opt}" in i) run_id=${OPTARG} ;; From f95b6a31221665f9a8c312ae2a0bf2be44e4df5f Mon Sep 17 00:00:00 2001 From: Aris Paschalidis Date: Tue, 2 Nov 2021 15:45:04 -0400 Subject: [PATCH 08/11] Format code --- MIPTools.def | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/MIPTools.def b/MIPTools.def index 9b32948..3ba2276 100644 --- a/MIPTools.def +++ b/MIPTools.def @@ -341,6 +341,7 @@ From: amd64/ubuntu:20.04 ## Basespace Download App ## ################################################################## %apprun bs_download + # Exit if something fails set -e # Set default values for paths @@ -382,11 +383,11 @@ From: amd64/ubuntu:20.04 # Argument handling using getopts # Could alternatively use manual processing. This would allow for long # form inputs. - while getopts 'i:o:c:h' opt; do + while getopts "i:o:c:h" opt; do case "${opt}" in i) run_id=${OPTARG} ;; o) ouput_path=${OPTARG} ;; - c) config_path="${OPTARG}" ;; + c) config_path=${OPTARG} ;; h) help exit 1 ;; *) help @@ -395,15 +396,16 @@ From: amd64/ubuntu:20.04 done # Ensure run_id is specified - if [ ! "$run_id" ]; then + if [ ! $run_id ]; then echo "Argument -i must be provided" - echo "$usage" >&2; exit 1 + echo "$usage" >&2 + exit 1 fi # Read data from config file # Remove whitespace from each line and export each line as a variable - export BASESPACE_API_SERVER=$(sed '1q;d' ${config_path} | sed 's/.*=.//g') - export BASESPACE_ACCESS_TOKEN=$(sed '2q;d' ${config_path} | sed 's/.*=.//g') + export BASESPACE_API_SERVER=$(sed "1q;d" ${config_path} | sed "s/.*=.//g") + export BASESPACE_ACCESS_TOKEN=$(sed "2q;d" ${config_path} | sed "s/.*=.//g") # Download data bs download run -i ${run_id} -o ${output_path} From 83b58dd684d53385a722e197a02800653d32c0d7 Mon Sep 17 00:00:00 2001 From: Aris Paschalidis Date: Tue, 2 Nov 2021 16:15:52 -0400 Subject: [PATCH 09/11] Don't require both `/opt/data` and `/opt/analysis` --- MIPTools.def | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MIPTools.def b/MIPTools.def index 3ba2276..2a4db95 100644 --- a/MIPTools.def +++ b/MIPTools.def @@ -422,7 +422,6 @@ From: amd64/ubuntu:20.04 s) sample_list=$OPTARG;; p) platform=$OPTARG;; ?) echo "Usage: singularity run --app demux \\" - echo " -B /path_to_run_dir:/opt/data \\" echo " -B /path_to_output_dir:/opt/analysis \\" echo " -B /path_to_base_resources:/opt/resources \\" echo " mycontainer.sif -s sample_list_file \\" @@ -448,7 +447,7 @@ From: amd64/ubuntu:20.04 "'"$output_dir"'")' # cd to where bcl files are. - cd /opt/data + cd /opt/analysis # Create a fastq directory for saving fastqs mkdir -p /opt/analysis/fastq From 8f5e9a7a8a27201361b5a6b623aac1efaea02e66 Mon Sep 17 00:00:00 2001 From: Aris Paschalidis Date: Tue, 2 Nov 2021 16:20:39 -0400 Subject: [PATCH 10/11] Style code --- MIPTools.def | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/MIPTools.def b/MIPTools.def index 2a4db95..fe1e200 100644 --- a/MIPTools.def +++ b/MIPTools.def @@ -414,13 +414,13 @@ From: amd64/ubuntu:20.04 ## Demux App ## ################################################################# %apprun demux - # Parse options - set -e - set -u - while getopts s:p: opt; do - case $opt in - s) sample_list=$OPTARG;; - p) platform=$OPTARG;; + # Exit if something fails or if have unset object + set -eu + + while getopts "s:p:" opt; do + case ${opt} in + s) sample_list=${OPTARG} ;; + p) platform=${OPTARG} ;; ?) echo "Usage: singularity run --app demux \\" echo " -B /path_to_output_dir:/opt/analysis \\" echo " -B /path_to_base_resources:/opt/resources \\" @@ -428,32 +428,32 @@ From: amd64/ubuntu:20.04 echo " -p sequencing_platform (nextseq or miseq) \\" echo "The sample list file must be present in the output" echo "directory mounted to /opt/analysis." - exit 1;; + exit 1 ;; esac done # Define variables cd /opt/src template_dir="/opt/resources/templates/sample_sheet_templates/" - platform_template="$platform"_sample_sheet_template.csv - template="$template_dir$platform_template" + platform_template="${platform}"_sample_sheet_template.csv + template="${template_dir}${platform_template}" bc_dict="/opt/resources/barcode_dict.json" output_dir="/opt/analysis" - sample_list="/opt/analysis/$sample_list" + sample_list="/opt/analysis/${sample_list}" # Create a sample sheet for demultiplexing python -c 'import mip_functions as mip; mip.generate_sample_sheet( "'"$sample_list"'", "'"$bc_dict"'", "'"$template"'", "'"$platform"'", "'"$output_dir"'")' - # cd to where bcl files are. + # cd to where bcl files are cd /opt/analysis # Create a fastq directory for saving fastqs mkdir -p /opt/analysis/fastq # Copy sample list to fastq directory - scp $sample_list /opt/analysis/fastq/ + scp ${sample_list} /opt/analysis/fastq/ # Increase limit of open number of files. ulimit -Sn $(ulimit -Hn) From 5e9921e1f1edabe7af24a42e12ee8234b9bd783a Mon Sep 17 00:00:00 2001 From: Aris Paschalidis Date: Tue, 2 Nov 2021 18:10:18 -0400 Subject: [PATCH 11/11] Update changelog --- CHANGELOG.md | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e5a5af..ffa2f1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,14 +2,14 @@ ## MIPTools (development version) -- Automatically build and deploy container using Github Actions (@arisp99, #11). -- Fix build failure due to dependency changes (#7). +### App Changes -### Maintenance - -- Remove duplicated files. -- Improve bash errors. -- Make strings human readable (@arisp99, #5). +- New `bs_download` app replaces the `download` app. The new app improves the + method to download data from the Illumina BaseSpace Sequence Hub by using + command line tools (@arisp99, #13). +- The `demux` app no longer requires both a run directory and a output + directory. These directories have been combined so that fastq files are + output to the run directory. ### Documentation Overhaul @@ -17,6 +17,17 @@ - Improve clarity of README and add additional instructions on downloading or building the container. +### Bug Fixes + +- Fix build failure due to dependency changes (#7). + +### Maintenance + +- Automatically build and deploy container using Github Actions (@arisp99, #11). +- Remove duplicated files. +- Improve bash errors. +- Make strings human readable (@arisp99, #5). + ## MIPTools 1.0.0 - First major release.