From d3705bf1f00cf4c47e7a678d0cbbe2bfe66a1ae6 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Tue, 18 Oct 2016 11:20:25 -0700 Subject: [PATCH 01/22] Try to set up travis to run unit tests --- .travis.yml | 5 ++ runtests.sh | 17 +++++ test/dotbasespace/completion.bash | 83 +++++++++++++++++++++++ test/dotbasespace/default-plugins.json | 1 + test/dotbasespace/default.cfg | 3 + test/dotbasespace/unit_tests-plugins.json | 1 + test/dotbasespace/unit_tests.cfg | 3 + 7 files changed, 113 insertions(+) create mode 100644 .travis.yml create mode 100755 runtests.sh create mode 100644 test/dotbasespace/completion.bash create mode 100644 test/dotbasespace/default-plugins.json create mode 100644 test/dotbasespace/default.cfg create mode 100644 test/dotbasespace/unit_tests-plugins.json create mode 100644 test/dotbasespace/unit_tests.cfg diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..ae48602 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,5 @@ +language: python +python: + - "2.7" +install: "cd src && python setup.py install && cd .." +script: ./runtests.sh diff --git a/runtests.sh b/runtests.sh new file mode 100755 index 0000000..9481115 --- /dev/null +++ b/runtests.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +mkdir ~/.basespace + +cp test/dotbasespace/*.bash ~/.basespace +cp test/dotbasespace/*.json ~/.basespace + +cd data +curl -O https://s3.amazonaws.com/basespace-sdk-unit-test-data/BC-12_S12_L001_R2_001.fastq.gz +cd .. + +# ... +cat test/dotbasespace/unit_tests.cfg | sed "s/__ACCESS_TOKEN__/$ACCESS_TOKEN/" > ~/.basespace/unit_tests.cfg +cp ~/.basespace/unit_tests.cfg ~/.basespace/default.cfg + + +python test/unit_tests.py diff --git a/test/dotbasespace/completion.bash b/test/dotbasespace/completion.bash new file mode 100644 index 0000000..2320be0 --- /dev/null +++ b/test/dotbasespace/completion.bash @@ -0,0 +1,83 @@ +_bs() +{ + local cur prev words + IFS=$'\n' + COMPREPLY=() + _get_comp_words_by_ref -n : cur prev words + + # Command data: + cmds=$'app\nappsession\nauth\nauthenticate\ncomplete\ncp\ncreate\nhelp\nhistory\nimport\nkill\nlaunch\nlist\nmount\nproject\nregister\nsample\nunmount\nunregister\nupload\nversion\nwhoami' + cmds_app=$'import launch' + cmds_app_import=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-i\n--appid\n-n\n--appname\n-p\n--properties-file\n-e\n--defaults-file\n-a\n--appsession-id\n-r\n--appversion\n-m\n--appsession-path\n-j\n--input-templates\n-f\n--force' + cmds_app_launch=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-i\n--appid\n-a\n--agentid\n-b\n--batch-size\n-n\n--appname\n-o\n--option\n-s\n--sample-attributes\n--disable-consistency-checking' + cmds_appsession=$'kill' + cmds_appsession_kill=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse' + cmds_auth=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n--api-server\n--api-version\n--force\n--scopes\n--client-id\n--client-secret' + cmds_authenticate=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n--api-server\n--api-version\n--force\n--scopes\n--client-id\n--client-secret' + cmds_complete=$'-h\n--help\n--name\n--shell' + cmds_cp=$'-h\n--help' + cmds_create=$'project' + cmds_create_project=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse' + cmds_help=$'-h\n--help' + cmds_history=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n--json\n--domain' + cmds_import=$'app' + cmds_import_app=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-i\n--appid\n-n\n--appname\n-p\n--properties-file\n-e\n--defaults-file\n-a\n--appsession-id\n-r\n--appversion\n-m\n--appsession-path\n-j\n--input-templates\n-f\n--force' + cmds_kill=$'appsession' + cmds_kill_appsession=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse' + cmds_launch=$'app' + cmds_launch_app=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-i\n--appid\n-a\n--agentid\n-b\n--batch-size\n-n\n--appname\n-o\n--option\n-s\n--sample-attributes\n--disable-consistency-checking' + cmds_mount=$'-h\n--help' + cmds_project=$'create' + cmds_project_create=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse' + cmds_register=$'-h\n--help\n-p\n--path\n-d\n--description\n-g\n--group\n--force' + cmds_sample=$'upload' + cmds_sample_upload=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-p\n--project\n-i\n--sample-id\n-e\n--experiment\n--show-validation-rules' + cmds_unmount=$'-h\n--help' + cmds_unregister=$'-h\n--help\n-g\n--group' + cmds_upload=$'sample' + cmds_upload_sample=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-p\n--project\n-i\n--sample-id\n-e\n--experiment\n--show-validation-rules' + cmds_version=$'-h\n--help' + cmds_list=$'samples\nprojects\nappsessions\nappresults\napps' + cmds_list_samples=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-f\n--format\n-C\n--column\n--max-width\n--noindent\n--quote\n--project-name' + cmds_list_projects=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-f\n--format\n-C\n--column\n--max-width\n--noindent\n--quote\n--project-name' + cmds_list_appsessions=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-f\n--format\n-C\n--column\n--max-width\n--noindent\n--quote\n--project-name' + cmds_list_appresults=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-f\n--format\n-C\n--column\n--max-width\n--noindent\n--quote\n--project-name' + cmds_list_apps=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-f\n--format\n-C\n--column\n--max-width\n--noindent\n--quote\n--project-name' + cmds_whoami=$'-v\n--verbose\n--log-file\n-q\n--quiet\n-h\n--help\n--debug\n-V\n--version\n--dry-run\n-c\n--config\n--terse\n-f\n--format\n-C\n--column\n--max-width\n--noindent\n--quote' + + cmd="" + words[0]="" + completed="${cmds}" + for var in "${words[@]:1}" + do + if [[ ${var} == -* ]] ; then + break + fi + if [ -z "${cmd}" ] ; then + proposed="${var}" + else + proposed="${cmd}_${var}" + fi + local i="cmds_${proposed}" + local comp="${!i}" + if [ -z "${comp}" ] ; then + break + fi + if [[ ${comp} == -* ]] ; then + if [[ ${cur} != -* ]] ; then + completed="" + break + fi + fi + cmd="${proposed}" + completed="${comp}" + done + + if [ -z "${completed}" ] ; then + COMPREPLY=( $( compgen -f -- "$cur" ) $( compgen -d -- "$cur" ) ) + else + COMPREPLY=( $(compgen -W "${completed}" -- ${cur}) ) + fi + return 0 +} +complete -o filenames -F _bs bs diff --git a/test/dotbasespace/default-plugins.json b/test/dotbasespace/default-plugins.json new file mode 100644 index 0000000..a9fcbee --- /dev/null +++ b/test/dotbasespace/default-plugins.json @@ -0,0 +1 @@ +{"third party": {}} \ No newline at end of file diff --git a/test/dotbasespace/default.cfg b/test/dotbasespace/default.cfg new file mode 100644 index 0000000..b770282 --- /dev/null +++ b/test/dotbasespace/default.cfg @@ -0,0 +1,3 @@ +[DEFAULT] +apiServer = https://api.cloud-hoth.illumina.com/ +accessToken = __ACCESS_TOKEN__ diff --git a/test/dotbasespace/unit_tests-plugins.json b/test/dotbasespace/unit_tests-plugins.json new file mode 100644 index 0000000..a9fcbee --- /dev/null +++ b/test/dotbasespace/unit_tests-plugins.json @@ -0,0 +1 @@ +{"third party": {}} \ No newline at end of file diff --git a/test/dotbasespace/unit_tests.cfg b/test/dotbasespace/unit_tests.cfg new file mode 100644 index 0000000..b770282 --- /dev/null +++ b/test/dotbasespace/unit_tests.cfg @@ -0,0 +1,3 @@ +[DEFAULT] +apiServer = https://api.cloud-hoth.illumina.com/ +accessToken = __ACCESS_TOKEN__ From 36ab562423aaba53c05a5f46bbe07c873e729abb Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Tue, 18 Oct 2016 11:25:06 -0700 Subject: [PATCH 02/22] add travis build badge to README this will need to change when PR is accepted, change `dtenenba` to `basespace`. And when `develop` is merged into `master` you can remove the `?branch=develop` bit. --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 351a126..7d47e84 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ -INTRODUCTION +![Build Status](https://api.travis-ci.org/dtenenba/basespace-python-sdk.svg?branch=develop) + + +INTRODUCTION ========================================= -BaseSpacePy is a Python based SDK to be used in the development of Apps and scripts for working with Illumina's BaseSpace cloud-computing solution for next-gen sequencing data analysis. +BaseSpacePy is a Python based SDK to be used in the development of Apps and scripts for working with Illumina's BaseSpace cloud-computing solution for next-gen sequencing data analysis. The primary purpose of the SDK is to provide an easy-to-use Python environment enabling developers to authenticate a user, retrieve data, and upload data/results from their own analysis to BaseSpace. @@ -39,7 +42,7 @@ If you do not have root access, you may use the --prefix to specify the install python setup.py install --prefix=/folder/in/my/pythonpath -For more install options type: +For more install options type: python setup.py --help @@ -53,7 +56,7 @@ or add it to the PYTHONPATH at the top of your Python scripts using BaseSpacePy: sys.path.append('/my/path/basespace-python-sdk/src') import BaseSpacePy -To test that everything is working as expected, launch a Python prompt and try importing 'BaseSpacePy': +To test that everything is working as expected, launch a Python prompt and try importing 'BaseSpacePy': mkallberg@ubuntu:~/$ python >>> import BaseSpacePy @@ -115,7 +118,7 @@ Update to support changes in BaseSpace REST specification version v1pre3. Specif v 0.1 ----------------------------------------- - + Initial release of BaseSpacePy COPYING / LICENSE From 9b21ce11c9029d9dab958ba2f6022f6b16cf7756 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Tue, 18 Oct 2016 11:33:24 -0700 Subject: [PATCH 03/22] ignore file downloaded for unit tests --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1463c0f..d31d26f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ parts var sdist /scripts +data/BC-12_S12_L001_R2_001.fastq.gz From e799e4044435bec412d415fa6a44e791759a1fb2 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Tue, 18 Oct 2016 14:56:43 -0700 Subject: [PATCH 04/22] some unit tests pass in py3 (all still pass in py2) --- .travis.yml | 1 + doc/_update_doc/conf.py | 16 +- examples/0_Browsing.py | 34 +- examples/1_AccessingFiles.py | 46 +- examples/2_AppTriggering.py | 44 +- examples/3_Authentication.py | 66 +- examples/4_AppResultUpload.py | 36 +- examples/5_Purchasing.py | 38 +- src/BaseSpacePy/api/APIClient.py | 117 +- src/BaseSpacePy/api/AppLaunchHelpers.py | 4 +- src/BaseSpacePy/api/AuthenticationAPI.py | 22 +- src/BaseSpacePy/api/BaseAPI.py | 81 +- src/BaseSpacePy/api/BaseMountInterface.py | 6 +- src/BaseSpacePy/api/BaseSpaceAPI.py | 565 ++++---- src/BaseSpacePy/api/BillingAPI.py | 6 +- src/BaseSpacePy/model/ListResponse.py | 4 +- .../model/MultipartFileTransfer.py | 280 ++-- src/BaseSpacePy/model/QueryParameters.py | 40 +- .../model/QueryParametersPurchasedProduct.py | 16 +- src/setup.py | 9 +- test/launch_helpers_tests.py | 8 +- test/test_models.py | 200 +-- test/unit_tests.py | 1223 +++++++++-------- 23 files changed, 1446 insertions(+), 1416 deletions(-) diff --git a/.travis.yml b/.travis.yml index ae48602..7f5dca4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ language: python python: - "2.7" + - "3.5" install: "cd src && python setup.py install && cd .." script: ./runtests.sh diff --git a/doc/_update_doc/conf.py b/doc/_update_doc/conf.py index c9f4e21..d75b995 100644 --- a/doc/_update_doc/conf.py +++ b/doc/_update_doc/conf.py @@ -40,8 +40,8 @@ master_doc = 'index' # General information about the project. -project = u'BaseSpacePy' -copyright = u'2014, Illumina' +project = 'BaseSpacePy' +copyright = '2014, Illumina' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -183,8 +183,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'BaseSpacePy.tex', u'BaseSpacePy Documentation', - u'Illumina', 'manual'), + ('index', 'BaseSpacePy.tex', 'BaseSpacePy Documentation', + 'Illumina', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -213,8 +213,8 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'basespacepy', u'BaseSpacePy Documentation', - [u'Illumina'], 1) + ('index', 'basespacepy', 'BaseSpacePy Documentation', + ['Illumina'], 1) ] # If true, show URL addresses after external links. @@ -227,8 +227,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'BaseSpacePy', u'BaseSpacePy Documentation', - u'Illumina', 'BaseSpacePy', 'One line description of project.', + ('index', 'BaseSpacePy', 'BaseSpacePy Documentation', + 'Illumina', 'BaseSpacePy', 'One line description of project.', 'Miscellaneous'), ] diff --git a/examples/0_Browsing.py b/examples/0_Browsing.py index d90710e..9161ede 100644 --- a/examples/0_Browsing.py +++ b/examples/0_Browsing.py @@ -5,7 +5,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,11 +14,11 @@ """ from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI -import os +import os """ This script demonstrates basic browsing of BaseSpace objects once an access-token -for global browsing has been obtained. +for global browsing has been obtained. """ """ @@ -30,8 +30,8 @@ [DEFAULT] name = my new app clientKey = -clientSecret = -accessToken = +clientSecret = +accessToken = appSessionId = apiServer = https://api.cloud-hoth.illumina.com/ apiVersion = v1pre3 @@ -49,35 +49,35 @@ myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId) else: myAPI = BaseSpaceAPI(profile='DEFAULT') - + # First, let's grab the genome with id=4 myGenome = myAPI.getGenomeById('4') -print "\nThe Genome is " + str(myGenome) -print "We can get more information from the genome object" -print 'Id: ' + myGenome.Id -print 'Href: ' + myGenome.Href -print 'DisplayName: ' + myGenome.DisplayName +print("\nThe Genome is " + str(myGenome)) +print("We can get more information from the genome object") +print('Id: ' + myGenome.Id) +print('Href: ' + myGenome.Href) +print('DisplayName: ' + myGenome.DisplayName) # Get a list of all genomes allGenomes = myAPI.getAvailableGenomes() -print "\nGenomes \n" + str(allGenomes) +print("\nGenomes \n" + str(allGenomes)) # Let's have a look at the current user user = myAPI.getUserById('current') -print "\nThe current user is \n" + str(user) +print("\nThe current user is \n" + str(user)) # Now list the projects for this user myProjects = myAPI.getProjectByUser() -print "\nThe projects for this user are \n" + str(myProjects) +print("\nThe projects for this user are \n" + str(myProjects)) # We can also achieve this by making a call using the 'user instance' myProjects2 = user.getProjects(myAPI) -print "\nProjects retrieved from the user instance \n" + str(myProjects2) +print("\nProjects retrieved from the user instance \n" + str(myProjects2)) # List the runs available for the current user runs = user.getRuns(myAPI) -print "\nThe runs for this user are \n" + str(runs) +print("\nThe runs for this user are \n" + str(runs)) # In the same manner we can get a list of accessible user runs runs = user.getRuns(myAPI) -print "\nRuns retrieved from user instance \n" + str(runs) +print("\nRuns retrieved from user instance \n" + str(runs)) diff --git a/examples/1_AccessingFiles.py b/examples/1_AccessingFiles.py index 7b65b1e..4d52a0d 100644 --- a/examples/1_AccessingFiles.py +++ b/examples/1_AccessingFiles.py @@ -5,7 +5,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,14 +16,14 @@ from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI import os """ -This script demonstrates how to access Samples and AppResults from a projects and how to work with the available -file data for such instances. +This script demonstrates how to access Samples and AppResults from a projects and how to work with the available +file data for such instances. """ """ -NOTE: The coverage and variants API calls below require access to a public +NOTE: The coverage and variants API calls below require access to a public dataset. Before running the example, first go to cloud-hoth.illumina.com, -login, click on Public Data, select the dataset named 'MiSeq B. cereus demo +login, click on Public Data, select the dataset named 'MiSeq B. cereus demo data', and click the Import button for the Project named 'BaseSpaceDemo'. """ @@ -51,40 +51,40 @@ # Let's list all the AppResults and samples for these projects for singleProject in myProjects: - print "# " + str(singleProject) + print("# " + str(singleProject)) appResults = singleProject.getAppResults(myAPI) - print " The App results for project " + str(singleProject) + " are \n\t" + str(appResults) + print(" The App results for project " + str(singleProject) + " are \n\t" + str(appResults)) samples = singleProject.getSamples(myAPI) - print " The samples for project " + str(singleProject) + " are \n\t" + str(samples) + print(" The samples for project " + str(singleProject) + " are \n\t" + str(samples)) # -## we'll take a further look at the files belonging to the sample and -##analyses from the last project in the loop above +## we'll take a further look at the files belonging to the sample and +##analyses from the last project in the loop above for a in appResults: - print "# " + a.Id + print("# " + a.Id) ff = a.getFiles(myAPI) - print ff + print(ff) for s in samples: - print "Sample " + str(s) + print("Sample " + str(s)) ff = s.getFiles(myAPI) - print ff + print(ff) -## Now let's do some work with files -## we'll grab a BAM by id and get the coverage for an interval + accompanying meta-data +## Now let's do some work with files +## we'll grab a BAM by id and get the coverage for an interval + accompanying meta-data myBam = myAPI.getFileById('9895890') -print myBam +print(myBam) cov = myBam.getIntervalCoverage(myAPI,'chr','1','100') -print cov +print(cov) try: covMeta = myBam.getCoverageMeta(myAPI,'chr') except Exception as e: - print "Coverage metadata may not be available for this BAM file: %s" % str(e) + print("Coverage metadata may not be available for this BAM file: %s" % str(e)) else: - print covMeta + print(covMeta) # ## and a vcf file myVCF = myAPI.getFileById('9895892') varMeta = myVCF.getVariantMeta(myAPI) -print varMeta -var = myVCF.filterVariant(myAPI,'chr','1', '25000') -print var +print(varMeta) +var = myVCF.filterVariant(myAPI,'chr','1', '25000') +print(var) diff --git a/examples/2_AppTriggering.py b/examples/2_AppTriggering.py index e38c87f..8ec00da 100644 --- a/examples/2_AppTriggering.py +++ b/examples/2_AppTriggering.py @@ -5,7 +5,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,7 +18,7 @@ import time """ -This script demonstrates how to retrieve the AppSession object produced +This script demonstrates how to retrieve the AppSession object produced when a user initiates an app. Further it's demonstrated how to automatically generate the scope strings to request access to the data object (a project or a sample) that the app was triggered to analyze. @@ -51,30 +51,30 @@ # Using the basespaceApi we can request the appSession object corresponding to the AppSession id supplied myAppSession = myAPI.getAppSession() -print myAppSession +print(myAppSession) # An app session contains a referal to one or more appLaunchObjects which reference the data module -# the user launched the app on. This can be a list of projects, samples, or a mixture of objects -print "\nType of data the app was triggered on can be seen in 'references'" -print myAppSession.References +# the user launched the app on. This can be a list of projects, samples, or a mixture of objects +print("\nType of data the app was triggered on can be seen in 'references'") +print(myAppSession.References) # We can also get a handle to the user who started the AppSession -print "\nWe can get a handle for the user who triggered the app\n" + str(myAppSession.UserCreatedBy) +print("\nWe can get a handle for the user who triggered the app\n" + str(myAppSession.UserCreatedBy)) # Let's have a closer look at the appSessionLaunchObject myReference = myAppSession.References[0] -print "\nWe can get out information such as the href to the launch object:" -print myReference.HrefContent -print "\nand the specific type of that object:" -print myReference.Type +print("\nWe can get out information such as the href to the launch object:") +print(myReference.HrefContent) +print("\nand the specific type of that object:") +print(myReference.Type) # Now we will want to ask for more permission for the specific reference object -print "\nWe can get out the specific project objects by using 'content':" +print("\nWe can get out the specific project objects by using 'content':") myReference = myReference.Content -print myReference -print "\nThe scope string for requesting read access to the reference object is:" -print myReference.getAccessStr(scope='write') +print(myReference) +print("\nThe scope string for requesting read access to the reference object is:") +print(myReference.getAccessStr(scope='write')) # We can easily request write access to the reference object so our App can start contributing analysis # by default we ask for write permission and authentication for a device @@ -82,21 +82,21 @@ # We may limit our request to read access only if that's all that is needed #readAccessMaps = myAPI.getAccess(myReference,accessType='read') -#print "\nWe get the following access map for the write request" -#print accessMap +#print("\nWe get the following access map for the write request") +#print(accessMap) ## PAUSE HERE # Have the user visit the verification uri to grant us access -#print "\nPlease visit the uri within 15 seconds and grant access" -#print accessMap['verification_with_code_uri'] +#print("\nPlease visit the uri within 15 seconds and grant access") +#print(accessMap['verification_with_code_uri']) #webbrowser.open_new(accessMap['verification_with_code_uri']) #time.sleep(15) ## PAUSE HERE # Once the user has granted us the access to the object we requested we can # get the basespace access token and start browsing simply by calling updatePriviliges -# on the baseSpaceApi instance +# on the baseSpaceApi instance #code = accessMap['device_code'] #myAPI.updatePrivileges(code) -#print "\nThe BaseSpaceAPI instance was update with write privileges" -#print myAPI +#print("\nThe BaseSpaceAPI instance was update with write privileges") +#print(myAPI) diff --git a/examples/3_Authentication.py b/examples/3_Authentication.py index 6b13753..ef15e99 100644 --- a/examples/3_Authentication.py +++ b/examples/3_Authentication.py @@ -5,26 +5,31 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import six +if six.PY2: + from __future__ import print_function import sys from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI import time -import webbrowser -import cPickle as Pickle +import webbrowser import os +from six.moves import cPickle as Pickle + + """ -Demonstrates the basic BaseSpace authentication process The work-flow is as follows: +Demonstrates the basic BaseSpace authentication process The work-flow is as follows: scope-request -> user grants access -> browsing data. The scenario is demonstrated both for device and web-based apps. -Further we demonstrate how a BaseSpaceAPI instance may be preserved across multiple http-request for the same app session using +Further we demonstrate how a BaseSpaceAPI instance may be preserved across multiple http-request for the same app session using python's pickle module. """ @@ -45,18 +50,18 @@ myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId) else: myAPI = BaseSpaceAPI(profile='DEFAULT') - - + + # First, get the verification code and uri for scope 'browse global' deviceInfo = myAPI.getVerificationCode('browse global') -print "\n URL for user to visit and grant access: " -print deviceInfo['verification_with_code_uri'] +print("\n URL for user to visit and grant access: ") +print(deviceInfo['verification_with_code_uri']) ## PAUSE HERE # Have the user visit the verification uri to grant us access -print "\nPlease visit the uri within 15 seconds and grant access" -print deviceInfo['verification_with_code_uri'] +print("\nPlease visit the uri within 15 seconds and grant access") +print(deviceInfo['verification_with_code_uri']) webbrowser.open_new(deviceInfo['verification_with_code_uri']) time.sleep(15) ## PAUSE HERE @@ -68,13 +73,13 @@ myAPI.updatePrivileges(code) # As a reference the provided access-token can be obtained from the BaseSpaceApi object -print "\nMy Access-token:" -print myAPI.getAccessToken() +print("\nMy Access-token:") +print(myAPI.getAccessToken()) -# Let's try and grab all available genomes with our new api! +# Let's try and grab all available genomes with our new api! allGenomes = myAPI.getAvailableGenomes() -print "\nGenomes \n" + str(allGenomes) +print("\nGenomes \n" + str(allGenomes)) # If at a later stage we wish to initialize a BaseSpaceAPI object when we already have @@ -82,8 +87,8 @@ # object using the key-word AccessToken. myToken = myAPI.getAccessToken() myAPI.setAccessToken(myToken) -print "\nA BaseSpaceAPI instance was updated with an access-token: " -print myAPI +print("\nA BaseSpaceAPI instance was updated with an access-token: ") +print(myAPI) #################### Web-based verification ################################# # The scenario where the authentication is done through a web-browser @@ -94,8 +99,8 @@ BSapiWeb = BaseSpaceAPI(profile='DEFAULT') userUrl= BSapiWeb.getWebVerificationCode('browse global','http://localhost',state='myState') -print "\nHave the user visit:" -print userUrl +print("\nHave the user visit:") +print(userUrl) webbrowser.open_new(userUrl) @@ -109,20 +114,20 @@ #################### Storing BaseSpaceApi using python's pickle module ################################# """ -It may sometimes be useful to preserve certain api objects across a series of http requests from the same user-session. +It may sometimes be useful to preserve certain api objects across a series of http requests from the same user-session. Here we demonstrate how the Python pickle module may be used to achieve this end. The example will be for an instance of BaseSpaceAPI, but the same technique may be used for BaseSpaceAuth. -In fact, a single instance of BaseSpaceAuth would be enough for a single App and could be shared by all http-requests, as the identity of -this object is only given by the client_key and client_secret. +In fact, a single instance of BaseSpaceAuth would be enough for a single App and could be shared by all http-requests, as the identity of +this object is only given by the client_key and client_secret. (There is, of course, no problem in having multiple identical BaseSpaceAuth instances). """ # Get current user user= myAPI.getUserById('current') -print user -print myAPI +print(user) +print(myAPI) #### Here some work goes on @@ -134,9 +139,9 @@ Pickle.dump(myAPI, f) f.close() -# Imagine the current request is done, we will simulate this by deleting the api instance +# Imagine the current request is done, we will simulate this by deleting the api instance myAPI = None -print "\nTry printing the removed API, we get: " + str(myAPI) +print("\nTry printing the removed API, we get: " + str(myAPI)) # Next request in the session with id = id123 comes in @@ -145,10 +150,9 @@ f = open(mySessionId) myAPI = Pickle.load(f) f.close() - print - print "We got the API back!" - print myAPI + print() + print("We got the API back!") + print(myAPI) else: - print "Looks like we haven't stored anything for this session yet" + print("Looks like we haven't stored anything for this session yet") # create a BaseSpaceAPI for the first time - diff --git a/examples/4_AppResultUpload.py b/examples/4_AppResultUpload.py index e5c2017..b8119f1 100644 --- a/examples/4_AppResultUpload.py +++ b/examples/4_AppResultUpload.py @@ -5,7 +5,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,7 +18,7 @@ """ This script demonstrates how to create a new AppResults object, change its state -and upload result files to it and download files from it. +and upload result files to it and download files from it. """ @@ -43,47 +43,47 @@ # Now we'll do some work of our own. First get a project to work on # we'll need write permission, for the project we are working on -# meaning we will need get a new token and instantiate a new BaseSpaceAPI +# meaning we will need get a new token and instantiate a new BaseSpaceAPI p = myAPI.getProjectById('89') # Assuming we have write access to the project -# we will list the current App Results for the project +# we will list the current App Results for the project appRes = p.getAppResults(myAPI,statuses=['Running']) -print "\nThe current running AppResults are \n" + str(appRes) +print("\nThe current running AppResults are \n" + str(appRes)) # now let's do some work! # to create an appResults for a project, simply give the name and description appResults = p.createAppResult(myAPI,"testing","this is my results",appSessionId='') -print "\nSome info about our new app results" -print appResults -print appResults.Id -print "\nThe app results also comes with a reference to our AppSession" +print("\nSome info about our new app results") +print(appResults) +print(appResults.Id) +print("\nThe app results also comes with a reference to our AppSession") myAppSession = appResults.AppSession -print myAppSession +print(myAppSession) # we can change the status of our AppSession and add a status-summary as follows myAppSession.setStatus(myAPI,'needsattention',"We worked hard, but encountered some trouble.") -print "\nAfter a change of status of the app sessions we get\n" + str(myAppSession) +print("\nAfter a change of status of the app sessions we get\n" + str(myAppSession)) # we'll set our appSession back to running so we can do some more work myAppSession.setStatus(myAPI,'running',"Back on track") -### Let's list all AppResults again and see if our new object shows up +### Let's list all AppResults again and see if our new object shows up appRes = p.getAppResults(myAPI,statuses=['Running']) -print "\nThe updated app results are \n" + str(appRes) +print("\nThe updated app results are \n" + str(appRes)) appResult2 = myAPI.getAppResultById(appResults.Id) -print appResult2 +print(appResult2) -## Now we will make another AppResult +## Now we will make another AppResult ## and try to upload a file to it appResults2 = p.createAppResult(myAPI,"My second AppResult","This one I will upload to") appResults2.uploadFile(myAPI, '/home/mkallberg/Desktop/testFile2.txt', 'BaseSpaceTestFile.txt', '/mydir/', 'text/plain') -print "\nMy AppResult number 2 \n" + str(appResults2) +print("\nMy AppResult number 2 \n" + str(appResults2)) ## let's see if our new file made it appResultFiles = appResults2.getFiles(myAPI) -print "\nThese are the files in the appResult" -print appResultFiles +print("\nThese are the files in the appResult") +print(appResultFiles) f = appResultFiles[-1] # we can even download our newly uploaded file diff --git a/examples/5_Purchasing.py b/examples/5_Purchasing.py index d9fbea0..96a67f3 100644 --- a/examples/5_Purchasing.py +++ b/examples/5_Purchasing.py @@ -5,7 +5,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,7 +41,7 @@ if not client_key: raise Exception("Please fill in client values (in the script) before running the script") -# Create a client for making calls for this user session +# Create a client for making calls for this user session billAPI = BillingAPI(BaseSpaceStoreUrl, version, AppSessionId, AccessToken=accessToken) # create a non-consumable purchase @@ -49,43 +49,43 @@ # create a consumable purchase, and associated it with an AppSession # also add tags to provide (fake) details about the purchase -print "\nCreating purchase\n" +print("\nCreating purchase\n") purch = billAPI.createPurchase({'id':product_id,'quantity':4, 'tags':["test","test_tag"] }, AppSessionId) # record the purchase Id and RefundSecret for refunding later purchaseId = purch.Id refundSecret = purch.RefundSecret -print "Now complete the purchase in your web browser" -print "CLOSE the browser window/tab after you click 'Purchase' (and don't proceed into the app)" +print("Now complete the purchase in your web browser") +print("CLOSE the browser window/tab after you click 'Purchase' (and don't proceed into the app)") time.sleep(3) ## PAUSE HERE -print "Opening: " + purch.HrefPurchaseDialog +print("Opening: " + purch.HrefPurchaseDialog) webbrowser.open_new(purch.HrefPurchaseDialog) -print "Waiting 30 seconds..." +print("Waiting 30 seconds...") time.sleep(30) ## PAUSE HERE -print "\nConfirm the purchase" +print("\nConfirm the purchase") post_purch = billAPI.getPurchaseById(purchaseId) -print "The status of the purchase is now: " + post_purch.Status +print("The status of the purchase is now: " + post_purch.Status) -print "\nRefunding the Purchase" +print("\nRefunding the Purchase") # note we must use the same access token that was provided used for the purchase refunded_purchase = billAPI.refundPurchase(purchaseId, refundSecret, comment='the product did not function well as a frisbee') -print "\nGetting all purchases for the current user with the tags we used for the purchase above" +print("\nGetting all purchases for the current user with the tags we used for the purchase above") purch_prods = billAPI.getUserProducts(Id='current', queryPars=qpp( {'Tags':'test,test_tag'} )) if not len(purch_prods): - print "\nHmmm, didn't find any purchases with these tags. Did everything go OK above?\n" + print("\nHmmm, didn't find any purchases with these tags. Did everything go OK above?\n") else: - print "\nFor the first of these purchases:\n" - print "Purchase Name: " + purch_prods[0].Name - print "Purchase Price: " + purch_prods[0].Price - print "Purchase Quantity: " + purch_prods[0].Quantity - print "Tags: " + str(purch_prods[0].Tags) + print("\nFor the first of these purchases:\n") + print("Purchase Name: " + purch_prods[0].Name) + print("Purchase Price: " + purch_prods[0].Price) + print("Purchase Quantity: " + purch_prods[0].Quantity) + print("Tags: " + str(purch_prods[0].Tags)) # Get the refund status of the purchase - print "\nGetting the (refunded) Purchase we just made" + print("\nGetting the (refunded) Purchase we just made") get_purch = billAPI.getPurchaseById(purch_prods[0].PurchaseId) - print "Refund Status: " + get_purch.RefundStatus + "\n" + print("Refund Status: " + get_purch.RefundStatus + "\n") diff --git a/src/BaseSpacePy/api/APIClient.py b/src/BaseSpacePy/api/APIClient.py index d25eea6..5fb1be4 100644 --- a/src/BaseSpacePy/api/APIClient.py +++ b/src/BaseSpacePy/api/APIClient.py @@ -2,10 +2,7 @@ import sys import os import re -import urllib -import urllib2 import io -import cStringIO import json from subprocess import * import subprocess @@ -15,11 +12,16 @@ from BaseSpacePy.api.BaseSpaceException import RestMethodException, ServerResponseException +import six +from six import moves +from six.moves import urllib + + class APIClient: def __init__(self, AccessToken, apiServerAndVersion, userAgent=None, timeout=10): ''' Initialize the API instance - + :param AccessToken: an access token :param apiServerAndVersion: the URL of the BaseSpace api server with api version :param timeout: (optional) the timeout in seconds for each request made, default 10 @@ -32,7 +34,7 @@ def __init__(self, AccessToken, apiServerAndVersion, userAgent=None, timeout=10) def __forcePostCall__(self, resourcePath, postData, headers): ''' For forcing a REST POST request (seems to be used when POSTing with no post data) - + :param resourcePath: the url to call, including server address and api version :param postData: a dictionary of data to post :param headers: a dictionary of header key/values to include in call @@ -46,7 +48,7 @@ def __forcePostCall__(self, resourcePath, postData, headers): pass import logging logging.getLogger("requests").setLevel(logging.WARNING) - encodedPost = urllib.urlencode(postData) + encodedPost = urllib.parse.urlencode(postData) resourcePath = "%s?%s" % (resourcePath, encodedPost) response = requests.post(resourcePath, data=json.dumps(postData), headers=headers) return response.text @@ -54,9 +56,9 @@ def __forcePostCall__(self, resourcePath, postData, headers): def __putCall__(self, resourcePath, headers, data): ''' Performs a REST PUT call to the API server. - - :param resourcePath: the url to call, including server address and api version - :param headers: a dictionary of header key/values to include in call + + :param resourcePath: the url to call, including server address and api version + :param headers: a dictionary of header key/values to include in call :param transFile: the name of the file containing only data to be PUT :returns: server response (a string containing upload status message (from curl?) followed by json response) ''' @@ -64,7 +66,7 @@ def __putCall__(self, resourcePath, headers, data): # cmd = 'curl -H "x-access-token:' + self.apiKey + '" -H "Content-MD5:' + headers['Content-MD5'].strip() +'" -T "'+ transFile +'" -X PUT ' + resourcePath # p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) # output = p.stdout.read() - # print output + # print(output) # return output import requests put_headers = { @@ -79,12 +81,12 @@ def __putCall__(self, resourcePath, headers, data): def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None, forcePost=False): ''' Call a REST API and return the server response. - + An access token header is automatically added. If a Content-Type header isn't included, one will be added with 'application/json' (except for PUT and forcePost calls). Query parameters with values of None aren't sent to the server. Server errors are to be handled by the caller (returned response contains error codes/msgs). - + :param resourcePath: the url to call, not including server address and api version :param method: REST method, including GET, POST (and forcePost, see below), and PUT (DELETE not yet supported) :param queryParams: dictionary of query parameters to be added to url, except for forcePost where they are added as 'postData'; not used for PUT calls @@ -101,41 +103,42 @@ def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None if self.userAgent: headers['User-Agent'] = self.userAgent if headerParams: - for param, value in headerParams.iteritems(): + for param, value in six.iteritems(headerParams): headers[param] = value # specify the content type - if not headers.has_key('Content-Type') and not method=='PUT' and not forcePost: + if not 'Content-Type' in headers and not method=='PUT' and not forcePost: headers['Content-Type'] = 'application/json' - # include access token in header + # include access token in header headers['Authorization'] = 'Bearer ' + self.apiKey - + data = None if method == 'GET': if queryParams: # Need to remove None values, these should not be sent sentQueryParams = {} - for param, value in queryParams.iteritems(): + for param, value in six.iteritems(queryParams): if value != None: sentQueryParams[param] = value - url = url + '?' + urllib.urlencode(sentQueryParams) - request = urllib2.Request(url=url, headers=headers) + url = url + '?' + urllib.parse.urlencode(sentQueryParams) + request = urllib.request.Request(url=url, headers=headers) elif method in ['POST', 'PUT', 'DELETE']: if queryParams: # Need to remove None values, these should not be sent sentQueryParams = {} - for param, value in queryParams.iteritems(): + for param, value in six.iteritems(queryParams): if value != None: sentQueryParams[param] = value - forcePostUrl = url - url = url + '?' + urllib.urlencode(sentQueryParams) + forcePostUrl = url + url = url + '?' + urllib.parse.urlencode(sentQueryParams) data = postData if data: if type(postData) not in [str, int, float, bool]: data = json.dumps(postData) if not forcePost: - if data and not len(data): + if data and not len(data): data='\n' # temp fix, in case is no data in the file, to prevent post request from failing - request = urllib2.Request(url=url, headers=headers, data=data)#,timeout=self.timeout) + data = data.encode('utf-8') + request = urllib.request.Request(url=url, headers=headers, data=data)#,timeout=self.timeout) else: response = self.__forcePostCall__(forcePostUrl, sentQueryParams, headers) if method in ['PUT', 'DELETE']: @@ -149,49 +152,49 @@ def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None # Make the request if not forcePost and not method in ['PUT', 'DELETE']: # the normal case try: - response = urllib2.urlopen(request, timeout=self.timeout).read() - except urllib2.HTTPError as e: - response = e.read() # treat http error as a response (handle in caller) - except urllib2.URLError as e: - raise ServerResponseException('URLError: ' + str(e)) + response = urllib.request.urlopen(request, timeout=self.timeout).read().decode('utf-8') + except urllib.error.HTTPError as e: + response = e.read() # treat http error as a response (handle in caller) + except urllib.error.URLError as e: + raise ServerResponseException('URLError: ' + str(e)) try: data = json.loads(response) except ValueError as e: raise ServerResponseException('Error decoding json in server response') - return data + return data def deserialize(self, obj, objClass): """ Deserialize a JSON string into a BaseSpacePy object. :param obj: A dictionary (or object?) to be deserialized into a class (objClass); or a value to be passed into a new native python type (objClass) - :param objClass: A class object or native python type for the deserialized object, or a string of a class name or native python type. (eg, Project.Project, int, 'Project', 'int') + :param objClass: A class object or native python type for the deserialized object, or a string of a class name or native python type. (eg, Project.Project, int, 'Project', 'int') :returns: A deserialized object - """ + """ # Create an object class from objClass, if a string was passed in # Avoid native python types 'file' - if type(objClass) == str: + if type(objClass) == str: try: - if (not str(objClass)=='File'): + if (not str(objClass)=='File'): objClass = eval(objClass.lower()) else: objClass = eval(objClass + '.' + objClass) except NameError: # not a native type, must be model class objClass = eval(objClass + '.' + objClass) - + # Create an instance of the object class - # If the instance is a native python type, return it + # If the instance is a native python type, return it if objClass in [str, int, float, bool]: return objClass(obj) instance = objClass() - + # For every swaggerType in the instance that is also in the passed-in obj, # set the instance value for native python types, # or recursively deserialize class instances. # For dynamic types, substitute real class after looking up 'Type' value. # For lists, deserialize all members of a list, including lists of lists (though not list of list of list...). - # For datetimes, convert to a readable output string - for attr, attrType in instance.swaggerTypes.iteritems(): + # For datetimes, convert to a readable output string + for attr, attrType in six.iteritems(instance.swaggerTypes): if attr in obj: value = obj[attr] if attrType in ['str', 'int', 'float', 'bool']: @@ -199,52 +202,52 @@ def deserialize(self, obj, objClass): try: value = attrType(value) except UnicodeEncodeError: - value = unicode(value) - setattr(instance, attr, value) - elif attrType == 'DynamicType': + value = six.text_type(value) + setattr(instance, attr, value) + elif attrType == 'DynamicType': try: - model_name = instance._dynamicType[value['Type']] + model_name = instance._dynamicType[value['Type']] except KeyError: pass # suppress this warning, which is caused by a bug in BaseSpace - #warn("Warning - unrecognized dynamic type: " + value['Type']) + #warn("Warning - unrecognized dynamic type: " + value['Type']) else: setattr(instance, attr, self.deserialize(value, model_name)) elif 'list<' in attrType: match = re.match('list<(.*)>', attrType) - subClass = match.group(1) - subValues = [] + subClass = match.group(1) + subValues = [] # lists of dynamic type - if subClass == 'DynamicType': - for subValue in value: + if subClass == 'DynamicType': + for subValue in value: try: - new_type = instance._dynamicType[subValue['Type']] + new_type = instance._dynamicType[subValue['Type']] except KeyError: - pass + pass # suppress this warning, which is caused by a bug in BaseSpace - #warn("Warning - unrecognized (list of) dynamic types: " + subValue['Type']) + #warn("Warning - unrecognized (list of) dynamic types: " + subValue['Type']) else: - subValues.append(self.deserialize(subValue, new_type)) + subValues.append(self.deserialize(subValue, new_type)) setattr(instance, attr, subValues) # typical lists - else: + else: for subValue in value: subValues.append(self.deserialize(subValue, subClass)) setattr(instance, attr, subValues) # list of lists (e.g. map[] property type) elif 'listoflists<' in attrType: match = re.match('listoflists<(.*)>', attrType) - subClass = match.group(1) - outvals = [] + subClass = match.group(1) + outvals = [] for outval in value: invals = [] for inval in outval: invals.append(self.deserialize(inval, subClass)) outvals.append(invals) setattr(instance, attr, outvals) - - elif attrType=='dict': + + elif attrType=='dict': setattr(instance, attr, value) elif attrType=='datetime': dt = dateutil.parser.parse(value) diff --git a/src/BaseSpacePy/api/AppLaunchHelpers.py b/src/BaseSpacePy/api/AppLaunchHelpers.py index 7b856a3..5d95678 100644 --- a/src/BaseSpacePy/api/AppLaunchHelpers.py +++ b/src/BaseSpacePy/api/AppLaunchHelpers.py @@ -428,7 +428,7 @@ def make_launch_json(self, user_supplied_vars, launch_name, api_version, sample_ raise LaunchSpecificationException( "Compulsory variable(s) missing! (%s)" % str(required_vars - supplied_var_names)) if supplied_var_names - self.get_variable_requirements(): - print "warning! unused variable(s) specified: (%s)" % str(supplied_var_names - self.get_variable_requirements()) + print("warning! unused variable(s) specified: (%s)" % str(supplied_var_names - self.get_variable_requirements())) all_vars = copy.copy(self.defaults) all_vars.update(user_supplied_vars) self.resolve_list_variables(all_vars) @@ -457,7 +457,7 @@ def dump_property_information(self): dump all properties with their type and any default value for verbose usage information output """ - print self.format_property_information() + print(self.format_property_information()) def format_minimum_requirements(self): minimum_requirements = self.get_minimum_requirements() diff --git a/src/BaseSpacePy/api/AuthenticationAPI.py b/src/BaseSpacePy/api/AuthenticationAPI.py index f0cd2ab..a0101f1 100644 --- a/src/BaseSpacePy/api/AuthenticationAPI.py +++ b/src/BaseSpacePy/api/AuthenticationAPI.py @@ -1,6 +1,5 @@ import sys import time -import ConfigParser import getpass import os import requests @@ -11,6 +10,14 @@ except: pass import logging + +import six + +from six.moves import input + +from six.moves import configparser + + logging.getLogger("requests").setLevel(logging.WARNING) __author__ = 'psaffrey' @@ -47,9 +54,9 @@ def parse_config(self): parses the config_path or creates it if it doesn't exist :param config_path: path to config file - :return: ConfigParser object + :return: configparser object """ - self.config = ConfigParser.SafeConfigParser() + self.config = configparser.SafeConfigParser() self.config.optionxform = str if os.path.exists(self.config_path): self.config.read(self.config_path) @@ -86,7 +93,8 @@ def check_session_details(self): pass def set_session_details(self, config_path): - username = raw_input("username:") + inputfunc = raw_input if six.PY2 else input + username = inputfunc("username:") password = getpass.getpass() s, r = self.basespace_session(username, password) self.config.set(self.DEFAULT_CONFIG_NAME, self.SESSION_TOKEN_NAME, r.cookies[self.COOKIE_NAME]) @@ -129,7 +137,7 @@ def set_oauth_details(self, client_id, client_secret, scopes): raise AuthenticationException(msg) auth_url = payload["verification_with_code_uri"] auth_code = payload["device_code"] - print "please authenticate here: %s" % auth_url + print("please authenticate here: %s" % auth_url) # poll the token URL until we get the token token_payload = { "client_id": client_id, @@ -156,6 +164,6 @@ def set_oauth_details(self, client_id, client_secret, scopes): self.construct_default_config(self.api_server) if not access_token: raise Exception("problem obtaining token!") - print "Success!" + print("Success!") self.config.set(self.DEFAULT_CONFIG_NAME, self.ACCESS_TOKEN_NAME, access_token) - self.write_config() \ No newline at end of file + self.write_config() diff --git a/src/BaseSpacePy/api/BaseAPI.py b/src/BaseSpacePy/api/BaseAPI.py index dab6b2f..33a8cab 100644 --- a/src/BaseSpacePy/api/BaseAPI.py +++ b/src/BaseSpacePy/api/BaseAPI.py @@ -1,10 +1,6 @@ from pprint import pprint -import urllib2 import shutil -import urllib -import httplib -import cStringIO import json import os import inspect @@ -13,6 +9,11 @@ from BaseSpacePy.api.BaseSpaceException import * from BaseSpacePy.model import * from itertools import chain +from six import moves +from six.moves import urllib + +from six.moves import http_client + class BaseAPI(object): ''' @@ -23,7 +24,7 @@ def __init__(self, AccessToken, apiServerAndVersion, userAgent=None, timeout=10, ''' :param AccessToken: the current access token :param apiServerAndVersion: the api server URL with api version - :param timeout: (optional) the timeout in seconds for each request made, default 10 + :param timeout: (optional) the timeout in seconds for each request made, default 10 :param verbose: (optional) prints verbose output, default False ''' self.apiClient = APIClient(AccessToken, apiServerAndVersion, userAgent=userAgent, timeout=timeout) @@ -33,17 +34,17 @@ def __json_print__(self, label, var): try: prefix = " " * len(label) var_list = json.dumps(var,indent=4).split('\n') # ,ensure_ascii=False - print label + var_list[0] + print(label + var_list[0]) if len(var_list)>1: - print "\n".join( [prefix + s for s in var_list[1:]] ) + print("\n".join( [prefix + s for s in var_list[1:]] )) except UnicodeDecodeError: - pass # we could disable ascii-enforcing, as shown above, but + pass # we could disable ascii-enforcing, as shown above, but # this will massively increase the volume of logs def __singleRequest__(self, myModel, resourcePath, method, queryParams, headerParams, postData=None, forcePost=False): ''' Call a REST API and deserialize response into an object, handles errors from server. - + :param myModel: a Response object that includes a 'Response' swaggerType key with a value for the model type to return :param resourcePath: the api url path to call (without server and version) :param method: the REST method type, eg. GET @@ -56,27 +57,27 @@ def __singleRequest__(self, myModel, resourcePath, method, queryParams, headerPa :raises ServerResponseException: if server returns an error or has no response :returns: an instance of the Response model from the provided myModel ''' - if self.verbose: - print "" - print "* " + inspect.stack()[1][3] + " (" + str(method) + ")" # caller - print ' # Path: ' + str(resourcePath) - print ' # QPars: ' + str(queryParams) - print ' # Hdrs: ' + str(headerParams) - print ' # forcePost: ' + str(forcePost) + if self.verbose: + print("") + print("* " + inspect.stack()[1][3] + " (" + str(method) + ")") # caller + print(' # Path: ' + str(resourcePath)) + print(' # QPars: ' + str(queryParams)) + print(' # Hdrs: ' + str(headerParams)) + print(' # forcePost: ' + str(forcePost)) self.__json_print__(' # postData: ',postData) response = self.apiClient.callAPI(resourcePath, method, queryParams, postData, headerParams, forcePost=forcePost) if self.verbose: self.__json_print__(' # Response: ',response) - if not response: + if not response: raise ServerResponseException('No response returned') - if response.has_key('ResponseStatus'): - if response['ResponseStatus'].has_key('ErrorCode'): + if 'ResponseStatus' in response: + if 'ErrorCode' in response['ResponseStatus']: raise ServerResponseException(str(response['ResponseStatus']['ErrorCode'] + ": " + response['ResponseStatus']['Message'])) - elif response['ResponseStatus'].has_key('Message'): + elif 'Message' in response['ResponseStatus']: raise ServerResponseException(str(response['ResponseStatus']['Message'])) - elif response.has_key('ErrorCode'): + elif 'ErrorCode' in response: raise ServerResponseException(response["MessageFormatted"]) - + responseObject = self.apiClient.deserialize(response, myModel) if hasattr(responseObject, "Response"): return responseObject.Response @@ -98,15 +99,15 @@ def __listRequest__(self, myModel, resourcePath, method, queryParams, headerPara :param headerParams: a dictionary of header parameters :param sort: sort the outputs from the API to prevent race-conditions - :raises ServerResponseException: if server returns an error or has no response + :raises ServerResponseException: if server returns an error or has no response :returns: a list of instances of the provided model ''' - if self.verbose: - print "" - print "* " + inspect.stack()[1][3] + " (" + str(method) + ")" # caller - print ' # Path: ' + str(resourcePath) - print ' # QPars: ' + str(queryParams) - print ' # Hdrs: ' + str(headerParams) + if self.verbose: + print("") + print("* " + inspect.stack()[1][3] + " (" + str(method) + ")") # caller + print(' # Path: ' + str(resourcePath)) + print(' # QPars: ' + str(queryParams)) + print(' # Hdrs: ' + str(headerParams)) number_received = 0 total_number = None responses = [] @@ -125,9 +126,9 @@ def __listRequest__(self, myModel, resourcePath, method, queryParams, headerPara self.__json_print__(' # Response: ',response) if not response: raise ServerResponseException('No response returned') - if response['ResponseStatus'].has_key('ErrorCode'): + if 'ErrorCode' in response['ResponseStatus']: raise ServerResponseException(str(response['ResponseStatus']['ErrorCode'] + ": " + response['ResponseStatus']['Message'])) - elif response['ResponseStatus'].has_key('Message'): + elif 'Message' in response['ResponseStatus']: raise ServerResponseException(str(response['ResponseStatus']['Message'])) respObj = self.apiClient.deserialize(response, ListResponse.ListResponse) @@ -161,7 +162,7 @@ def __makeCurlRequest__(self, data, url): if not r: raise ServerResponseException("No response from server") obj = json.loads(r.text) - if obj.has_key('error'): + if 'error' in obj: raise ServerResponseException(str(obj['error'] + ": " + obj['error_description'])) return obj @@ -174,21 +175,21 @@ def getTimeout(self): def setTimeout(self, time): ''' Specify the timeout in seconds for each request made - + :param time: timeout in seconds - ''' + ''' self.apiClient.timeout = time - + def getAccessToken(self): ''' - Returns the current access token. - ''' - return self.apiClient.apiKey + Returns the current access token. + ''' + return self.apiClient.apiKey def setAccessToken(self, token): ''' Sets the current access token. - + :param token: an access token ''' - self.apiClient.apiKey = token + self.apiClient.apiKey = token diff --git a/src/BaseSpacePy/api/BaseMountInterface.py b/src/BaseSpacePy/api/BaseMountInterface.py index 199f240..146b8d7 100644 --- a/src/BaseSpacePy/api/BaseMountInterface.py +++ b/src/BaseSpacePy/api/BaseMountInterface.py @@ -74,12 +74,12 @@ def __str__(self): return "%s : (%s) : (%s)" % (self.path, self.id, self.type) def _get_access_token_from_config(self, config_path): - from ConfigParser import SafeConfigParser, NoSectionError, NoOptionError + from configParser import SafeConfigParser, NoSectionError, NoOptionError config = SafeConfigParser() config.read(config_path) try: return config.get("DEFAULT", "accessToken") - except NoOptionError, NoSectionError: + except NoOptionError as NoSectionError: raise BaseMountInterfaceException("malformed BaseMount config: %s" % config_path) @@ -105,4 +105,4 @@ def get_meta_data(self): import sys path = sys.argv[1] mbi = BaseMountInterface(path) - print mbi \ No newline at end of file + print(mbi) diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index aaa7dec..561c053 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -1,17 +1,11 @@ from pprint import pprint -import urllib2 import shutil -import urllib -import httplib -import cStringIO import json import os import re from tempfile import mkdtemp import socket -import ConfigParser -import urlparse import logging import getpass import requests @@ -23,6 +17,13 @@ from BaseSpacePy.model.MultipartFileTransfer import MultipartDownload as mpd from BaseSpacePy.model.QueryParameters import QueryParameters as qp from BaseSpacePy.model import * +import six +from six import moves +from six.moves import http_client +from six.moves import urllib + +from six.moves import configparser + # Uris for obtaining a access token, user verification code, and app trigger information tokenURL = '/oauthv2/token' @@ -41,20 +42,20 @@ class BaseSpaceAPI(BaseAPI): ''' def __init__(self, clientKey=None, clientSecret=None, apiServer=None, version='v1pre3', appSessionId='', AccessToken='', userAgent=None, timeout=10, verbose=0, profile='default'): ''' - The following arguments are required in either the constructor or a config file (~/.basespacepy.cfg): - + The following arguments are required in either the constructor or a config file (~/.basespacepy.cfg): + :param clientKey: the client key of the user's app; required in constructor or config file :param clientSecret: the client secret of the user's app; required in constructor or config file :param apiServer: the URL of the BaseSpace api server; required in constructor or config file :param version: the version of the BaseSpace API; required in constructor or config file :param appSessionId: optional, though may be needed for AppSession-related methods :param AccessToken: optional, though will be needed for most methods (except to obtain a new access token) - :param timeout: optional, timeout period in seconds for api calls, default 10 + :param timeout: optional, timeout period in seconds for api calls, default 10 :param profile: optional, name of profile in config file, default 'DEFAULT' ''' - + cred = self._setCredentials(clientKey, clientSecret, apiServer, version, appSessionId, AccessToken, profile) - + self.appSessionId = cred['appSessionId'] self.key = cred['clientKey'] self.secret = cred['clientSecret'] @@ -65,7 +66,7 @@ def __init__(self, clientKey=None, clientSecret=None, apiServer=None, version='v # TODO this replacement won't work for all environments self.weburl = cred['apiServer'].replace('api.','') - apiServerAndVersion = urlparse.urljoin(cred['apiServer'], cred['apiVersion']) + apiServerAndVersion = urllib.parse.urljoin(cred['apiServer'], cred['apiVersion']) super(BaseSpaceAPI, self).__init__(cred['accessToken'], apiServerAndVersion, userAgent, timeout, verbose) def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSessionId, accessToken, profile): @@ -80,7 +81,7 @@ def _setCredentials(self, clientKey, clientSecret, apiServer, apiVersion, appSes :param apiServer: the URL of the BaseSpace api server :param version: the version of the BaseSpace API :param appSessionId: the AppSession Id - :param AccessToken: an access token + :param AccessToken: an access token :param profile: name of the config file :returns: dictionary with credentials from constructor, config file, or default (for optional args), in this priority order. ''' @@ -125,81 +126,81 @@ def _getLocalCredentials(self, profile): ''' Returns credentials from local config file (~/.basespace/.cfg) If some or all credentials are missing, they aren't included the in the returned dict - + :param profile: Profile name to use to find local config file - :returns: A dictionary with credentials from local config file + :returns: A dictionary with credentials from local config file ''' config_file = os.path.join(os.path.expanduser('~/.basespace'), "%s.cfg" % profile) if not os.path.exists(config_file): raise CredentialsException("Could not find config file: %s" % config_file) section_name = "DEFAULT" - cred = {} - config = ConfigParser.SafeConfigParser() + cred = {} + config = configparser.SafeConfigParser() if config.read(config_file): cred['name'] = profile try: cred['clientKey'] = config.get(section_name, "clientKey") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass try: cred['clientSecret'] = config.get(section_name, "clientSecret") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass try: cred['apiServer'] = config.get(section_name, "apiServer") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass try: cred['apiVersion'] = config.get(section_name, "apiVersion") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass - try: + try: cred['appSessionId'] = config.get(section_name, "appSessionId") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass try: cred['accessToken'] = config.get(section_name, "accessToken") - except ConfigParser.NoOptionError: - pass + except configparser.NoOptionError: + pass return cred def getAppSessionById(self, Id): ''' Get metadata about an AppSession. Note that the client key and secret must match those of the AppSession's Application. - + :param Id: The Id of the AppSession :returns: An AppSession instance - ''' + ''' return self.getAppSession(Id=Id) def getAppSessionOld(self, Id=None): ''' - Get metadata about an AppSession. - Note that the client key and secret must match those of the AppSession's Application. - - :param Id: an AppSession Id; if not provided, the AppSession Id of the BaseSpaceAPI instance will be used - :returns: An AppSession instance + Get metadata about an AppSession. + Note that the client key and secret must match those of the AppSession's Application. + + :param Id: an AppSession Id; if not provided, the AppSession Id of the BaseSpaceAPI instance will be used + :returns: An AppSession instance ''' if Id is None: Id = self.appSessionId if not Id: raise AppSessionException("An AppSession Id is required") - resourcePath = self.apiClient.apiServerAndVersion + '/appsessions/{AppSessionId}' - resourcePath = resourcePath.replace('{AppSessionId}', Id) - response = cStringIO.StringIO() + resourcePath = self.apiClient.apiServerAndVersion + '/appsessions/{AppSessionId}' + resourcePath = resourcePath.replace('{AppSessionId}', Id) + response = moves.cStringIO() import requests response = requests.get(resourcePath, auth=(self.key, self.secret)) resp_dict = json.loads(response.text) - return self.__deserializeAppSessionResponse__(resp_dict) + return self.__deserializeAppSessionResponse__(resp_dict) def getAppSession(self, Id=None, queryPars=None): if Id is None: Id = self.appSessionId if not Id: raise AppSessionException("An AppSession Id is required") - resourcePath = '/appsessions/{AppSessionId}' - resourcePath = resourcePath.replace('{AppSessionId}', Id) + resourcePath = '/appsessions/{AppSessionId}' + resourcePath = resourcePath.replace('{AppSessionId}', Id) method = 'GET' headerParams = {} queryParams = {} @@ -214,58 +215,58 @@ def getAllAppSessions(self, queryPars=None): def __deserializeAppSessionResponse__(self, response): ''' - Converts a AppSession response from the API server to an AppSession object. - + Converts a AppSession response from the API server to an AppSession object. + :param response: a dictionary (decoded from json) from getting an AppSession from the api server - :returns: An AppSession instance - ''' - if response['ResponseStatus'].has_key('ErrorCode'): - raise AppSessionException('BaseSpace error: ' + str(response['ResponseStatus']['ErrorCode']) + ": " + response['ResponseStatus']['Message']) + :returns: An AppSession instance + ''' + if 'ErrorCode' in response['ResponseStatus']: + raise AppSessionException('BaseSpace error: ' + str(response['ResponseStatus']['ErrorCode']) + ": " + response['ResponseStatus']['Message']) tempApi = APIClient(AccessToken='', apiServerAndVersion=self.apiClient.apiServerAndVersion, userAgent=self.apiClient.userAgent) - res = tempApi.deserialize(response, AppSessionResponse.AppSessionResponse) + res = tempApi.deserialize(response, AppSessionResponse.AppSessionResponse) return res.Response.__deserializeReferences__(self) def getAppSessionPropertiesById(self, Id, queryPars=None): ''' Returns the Properties of an AppSession - + :param Id: An AppSession Id :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering - :returns: A PropertyList instance - ''' - queryParams = self._validateQueryParameters(queryPars) + :returns: A PropertyList instance + ''' + queryParams = self._validateQueryParameters(queryPars) resourcePath = '/appsessions/{Id}/properties' resourcePath = resourcePath.replace('{Id}',Id) - method = 'GET' - headerParams = {} + method = 'GET' + headerParams = {} return self.__singleRequest__(PropertiesResponse.PropertiesResponse, resourcePath, method, queryParams, headerParams) def getAppSessionPropertyByName(self, Id, name, queryPars=None): ''' Returns the multi-value Property of the provided AppSession that has the provided Property name. Note - this method (and REST API) is supported for ONLY multi-value Properties. - + :param Id: The AppSessionId :param name: Name of the multi-value property to retrieve :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering - :returns: A multi-value propertylist instance such as MultiValuePropertyAppResultsList (depending on the Property Type) + :returns: A multi-value propertylist instance such as MultiValuePropertyAppResultsList (depending on the Property Type) ''' - queryParams = self._validateQueryParameters(queryPars) + queryParams = self._validateQueryParameters(queryPars) resourcePath = '/appsessions/{Id}/properties/{Name}/items' resourcePath = resourcePath.replace('{Id}', Id) - resourcePath = resourcePath.replace('{Name}', name) - method = 'GET' + resourcePath = resourcePath.replace('{Name}', name) + method = 'GET' headerParams = {} return self.__singleRequest__(MultiValuePropertyResponse.MultiValuePropertyResponse, resourcePath, method, queryParams, headerParams) - + def getAppSessionInputsById(self, Id, queryPars=None): ''' Returns the input properties of an AppSession - + :param Id: An AppSessionId :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering - :returns: a dictionary of input properties, keyed by input Name - ''' + :returns: a dictionary of input properties, keyed by input Name + ''' props = self.getAppSessionPropertiesById(Id, queryPars) inputs = {} for prop in props.Items: @@ -278,13 +279,13 @@ def setAppSessionState(self, Id, Status, Summary): ''' Set the Status and StatusSummary of an AppSession in BaseSpace. Note - once Status is set to Completed or Aborted, no further changes can made. - + :param Id: The id of the AppSession :param Status: The AppSession status string, must be one of: Running, Complete, NeedsAttention, TimedOut, Aborted :param Summary: The status summary string :returns: An updated AppSession instance ''' - resourcePath = '/appsessions/{Id}' + resourcePath = '/appsessions/{Id}' method = 'POST' resourcePath = resourcePath.replace('{Id}', Id) queryParams = {} @@ -311,7 +312,7 @@ def stopAppSession(self, Id): queryParams = {} headerParams = {} postData = {} - apiServerAndVersion = urlparse.urljoin(self.apiServer, "v2") + apiServerAndVersion = urllib.parse.urljoin(self.apiServer, "v2") v2api = BaseAPI(self.getAccessToken(), apiServerAndVersion) return v2api.__singleRequest__(AppSessionResponse.AppSessionResponse, resourcePath, method, queryParams, headerParams, postData=postData) @@ -320,8 +321,8 @@ def __deserializeObject__(self, dct, type): ''' Converts API response into object instances for Projects, Samples, and AppResults. For other types, the input value is simply returned. - - (Currently called by Sample's getReferencedAppResults() and + + (Currently called by Sample's getReferencedAppResults() and AppSessionLaunchObject's __deserializeObject__() to serialize References) :param dct: dictionary from an API response (converted from JSON) for a BaseSpace item (eg., a Project) @@ -334,13 +335,13 @@ def __deserializeObject__(self, dct, type): if type.lower()=='sample': return tempApi.deserialize(dct, Sample.Sample) if type.lower()=='appresult': - return tempApi.deserialize(dct, AppResult.AppResult) - return dct - + return tempApi.deserialize(dct, AppResult.AppResult) + return dct + def getAccess(self, obj, accessType='write', web=False, redirectURL='', state=''): ''' Requests access to the provided BaseSpace object. - + :param obj: The data object we wish to get access to -- must be a Project, Sample, AppResult, or Run. :param accessType: (Optional) the type of access (browse|read|write|create), default is write. Create is only supported for Projects. :param web: (Optional) true if the App is web-based, default is false meaning a device based app @@ -357,12 +358,12 @@ def getAccess(self, obj, accessType='write', web=False, redirectURL='', state='' return self.getWebVerificationCode(scopeStr, redirectURL, state) else: return self.getVerificationCode(scopeStr) - + def getVerificationCode(self, scope): ''' - For non-web applications (eg. devices), returns the device code - and verification url for the user to approve access to a specific data scope. - + For non-web applications (eg. devices), returns the device code + and verification url for the user to approve access to a specific data scope. + :param scope: The scope that access is requested for (e.g. 'browse project 123') :returns: dictionary of server response ''' @@ -372,21 +373,21 @@ def getVerificationCode(self, scope): def getWebVerificationCode(self, scope, redirectURL, state=''): ''' Generates the URL the user should be redirected to for web-based authentication - + :param scope: The scope that access is requested for (e.g. 'browse project 123') :param redirectURL: The redirect URL :param state: (Optional) A state parameter that will passed through to the redirect response - :returns: a url - ''' + :returns: a url + ''' data = {'client_id': self.key, 'redirect_uri': redirectURL, 'scope': scope, 'response_type': 'code', "state": state} - return self.weburl + webAuthorize + '?' + urllib.urlencode(data) + return self.weburl + webAuthorize + '?' + urllib.parse.urlencode(data) def obtainAccessToken(self, code, grantType='device', redirect_uri=None): ''' - Returns a user specific access token, for either device (non-web) or web apps. - + Returns a user specific access token, for either device (non-web) or web apps. + :param code: The device code returned by the getVerificationCode method - :param grantType: Grant-type may be either 'device' for non-web apps (default), or 'authorization_code' for web apps + :param grantType: Grant-type may be either 'device' for non-web apps (default), or 'authorization_code' for web apps :param redirect_uri: The uri to redirect to; required for web apps only. :raises OAuthException: when redirect_uri isn't provided by web apps :returns: an access token @@ -414,12 +415,12 @@ def getAccessTokenDetails(self): raise ServerResponseException('Could not query access token endpoint: %s' % str(e)) if not response: raise ServerResponseException('No response returned') - if response.has_key('ResponseStatus'): - if response['ResponseStatus'].has_key('ErrorCode'): + if 'ResponseStatus' in response: + if 'ErrorCode' in response['ResponseStatus']: raise ServerResponseException(str(response['ResponseStatus']['ErrorCode'] + ": " + response['ResponseStatus']['Message'])) - elif response['ResponseStatus'].has_key('Message'): + elif 'Message' in response['ResponseStatus']: raise ServerResponseException(str(response['ResponseStatus']['Message'])) - elif response.has_key('ErrorCode'): + elif 'ErrorCode' in response: raise ServerResponseException(response["MessageFormatted"]) responseObject = self.apiClient.deserialize(response["Response"], Token.Token) @@ -431,27 +432,27 @@ def updatePrivileges(self, code, grantType='device', redirect_uri=None): Retrieves a user-specific access token, and sets the token on the current object. :param code: The device code returned by the getVerificationCode method - :param grantType: Grant-type may be either 'device' for non-web apps (default), or 'authorization_code' for web apps + :param grantType: Grant-type may be either 'device' for non-web apps (default), or 'authorization_code' for web apps :param redirect_uri: The uri to redirect to; required for web apps only. - :returns: None + :returns: None ''' token = self.obtainAccessToken(code, grantType=grantType, redirect_uri=redirect_uri) self.setAccessToken(token) - + def createProject(self, Name): ''' - Creates a project with the specified name and returns a project object. + Creates a project with the specified name and returns a project object. If a project with this name already exists, the existing project is returned. - + :param Name: Name of the project - :returns: a Project instance of the newly created project - ''' - resourcePath = '/projects/' + :returns: a Project instance of the newly created project + ''' + resourcePath = '/projects/' method = 'POST' queryParams = {} headerParams = {} postData = {} - postData['Name'] = Name + postData['Name'] = Name return self.__singleRequest__(ProjectResponse.ProjectResponse, resourcePath, method, queryParams, headerParams, postData=postData) @@ -461,26 +462,26 @@ def launchApp(self, appId, configJson): queryParams = {} headerParams = { 'Content-Type' : "application/json" } postData = configJson - return self.__singleRequest__(AppLaunchResponse.AppLaunchResponse, + return self.__singleRequest__(AppLaunchResponse.AppLaunchResponse, resourcePath, method, queryParams, headerParams, postData=postData) def getUserById(self, Id): ''' Returns the User object corresponding to User Id - + :param Id: The Id of the user :returns: a User instance - ''' - resourcePath = '/users/{Id}' + ''' + resourcePath = '/users/{Id}' method = 'GET' resourcePath = resourcePath.replace('{Id}', Id) queryParams = {} headerParams = {} return self.__singleRequest__(UserResponse.UserResponse, resourcePath, method, queryParams, headerParams) - + def getAppResultFromAppSessionId(self, Id, appResultName=""): ''' - Returns an AppResult object from an AppSession Id. + Returns an AppResult object from an AppSession Id. if appResultName is supplied, look for an appresult with this name otherwise, expect there to be exactly one appresult @@ -503,47 +504,47 @@ def getAppResultFromAppSessionId(self, Id, appResultName=""): def getAppResultById(self, Id, queryPars=None): ''' Returns an AppResult object corresponding to Id - + :param Id: The Id of the AppResult :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: an AppResult instance - ''' + ''' queryParams = self._validateQueryParameters(queryPars) resourcePath = '/appresults/{Id}' resourcePath = resourcePath.replace('{format}', 'json') method = 'GET' - resourcePath = resourcePath.replace('{Id}', Id) + resourcePath = resourcePath.replace('{Id}', Id) headerParams = {} return self.__singleRequest__(AppResultResponse.AppResultResponse,resourcePath, method, queryParams, headerParams) def getAppResultPropertiesById(self, Id, queryPars=None): ''' Returns the Properties of an AppResult object corresponding to AppResult Id - + :param Id: The Id of the AppResult :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a PropertyList instance - ''' - queryParams = self._validateQueryParameters(queryPars) + ''' + queryParams = self._validateQueryParameters(queryPars) resourcePath = '/appresults/{Id}/properties' resourcePath = resourcePath.replace('{format}', 'json') method = 'GET' - resourcePath = resourcePath.replace('{Id}', Id) + resourcePath = resourcePath.replace('{Id}', Id) headerParams = {} return self.__singleRequest__(PropertiesResponse.PropertiesResponse, resourcePath, method, queryParams, headerParams) def getAppResultFilesById(self, Id, queryPars=None): ''' Returns a list of File object for an AppResult - + :param Id: The id of the AppResult :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering - :returns: a list of File instances + :returns: a list of File instances ''' - queryParams = self._validateQueryParameters(queryPars) + queryParams = self._validateQueryParameters(queryPars) resourcePath = '/appresults/{Id}/files' resourcePath = resourcePath.replace('{format}', 'json') - method = 'GET' + method = 'GET' headerParams = {} resourcePath = resourcePath.replace('{Id}',Id) return self.__listRequest__(File.File,resourcePath, method, queryParams, headerParams) @@ -551,12 +552,12 @@ def getAppResultFilesById(self, Id, queryPars=None): def getAppResultFiles(self, Id, queryPars=None): ''' * Deprecated in favor of getAppResultFileById() * - + Returns a list of File object for an AppResult - + :param Id: The id of the AppResult :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering - :returns: a list of File instances + :returns: a list of File instances ''' return self.getAppResultFilesById(Id, queryPars) @@ -586,46 +587,46 @@ def downloadAppResultFilesByExtension(self, Id, extension, localDir, appResultNa def getProjectById(self, Id, queryPars=None): ''' Request a project object by Id - + :param Id: The Id of the project :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a Project instance ''' - queryParams = self._validateQueryParameters(queryPars) + queryParams = self._validateQueryParameters(queryPars) resourcePath = '/projects/{Id}' resourcePath = resourcePath.replace('{format}', 'json') method = 'GET' - resourcePath = resourcePath.replace('{Id}', Id) + resourcePath = resourcePath.replace('{Id}', Id) headerParams = {} return self.__singleRequest__(ProjectResponse.ProjectResponse, resourcePath, method, queryParams, headerParams) def getProjectPropertiesById(self, Id, queryPars=None): ''' Request the Properties of a project object by Id - + :param Id: The Id of the project :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a ProjectList instance ''' - queryParams = self._validateQueryParameters(queryPars) + queryParams = self._validateQueryParameters(queryPars) resourcePath = '/projects/{Id}/properties' resourcePath = resourcePath.replace('{format}', 'json') method = 'GET' - resourcePath = resourcePath.replace('{Id}', Id) + resourcePath = resourcePath.replace('{Id}', Id) headerParams = {} return self.__singleRequest__(PropertiesResponse.PropertiesResponse,resourcePath, method, queryParams, headerParams) - + def getProjectByUser(self, queryPars=None): ''' Returns a list available projects for the current User. - + :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a list of Project instances ''' - queryParams = self._validateQueryParameters(queryPars) + queryParams = self._validateQueryParameters(queryPars) resourcePath = '/users/current/projects' resourcePath = resourcePath.replace('{format}', 'json') - method = 'GET' + method = 'GET' headerParams = {} return self.__listRequest__(Project.Project,resourcePath, method, queryParams, headerParams) @@ -644,92 +645,92 @@ def getUserProjectByName(self, projectName): def getAccessibleRunsByUser(self, queryPars=None): ''' Returns a list of accessible runs for the current User - + :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a list of Run instances - ''' - queryParams = self._validateQueryParameters(queryPars) + ''' + queryParams = self._validateQueryParameters(queryPars) resourcePath = '/users/current/runs' resourcePath = resourcePath.replace('{format}', 'json') - method = 'GET' + method = 'GET' headerParams = {} return self.__listRequest__(Run.Run, resourcePath, method, queryParams, headerParams) - + def getRunById(self, Id, queryPars=None): - ''' + ''' Request a run object by Id - + :param Id: The Id of the run :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a Run instance - ''' - queryParams = self._validateQueryParameters(queryPars) - resourcePath = '/runs/{Id}' + ''' + queryParams = self._validateQueryParameters(queryPars) + resourcePath = '/runs/{Id}' method = 'GET' - resourcePath = resourcePath.replace('{Id}', Id) + resourcePath = resourcePath.replace('{Id}', Id) headerParams = {} return self.__singleRequest__(RunResponse.RunResponse,resourcePath, method, queryParams, headerParams) - + def getRunPropertiesById(self, Id, queryPars=None): - ''' + ''' Request the Properties of a run object by Id - + :param Id: The Id of the run :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a PropertyList instance - ''' - queryParams = self._validateQueryParameters(queryPars) - resourcePath = '/runs/{Id}/properties' + ''' + queryParams = self._validateQueryParameters(queryPars) + resourcePath = '/runs/{Id}/properties' method = 'GET' - resourcePath = resourcePath.replace('{Id}', Id) + resourcePath = resourcePath.replace('{Id}', Id) headerParams = {} return self.__singleRequest__(PropertiesResponse.PropertiesResponse,resourcePath, method, queryParams, headerParams) def getRunFilesById(self, Id, queryPars=None): - ''' + ''' Request the files associated with a Run, using the Run's Id - + :param Id: The Id of the run :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a list of Run instances - ''' - queryParams = self._validateQueryParameters(queryPars) - resourcePath = '/runs/{Id}/files' + ''' + queryParams = self._validateQueryParameters(queryPars) + resourcePath = '/runs/{Id}/files' method = 'GET' - resourcePath = resourcePath.replace('{Id}', Id) - headerParams = {} + resourcePath = resourcePath.replace('{Id}', Id) + headerParams = {} return self.__listRequest__(File.File,resourcePath, method, queryParams, headerParams) def getRunSamplesById(self, Id, queryPars=None): - ''' + ''' Request the Samples associated with a Run, using the Run's Id - + :param Id: The Id of the run :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a list of Sample instances - ''' - queryParams = self._validateQueryParameters(queryPars) - resourcePath = '/runs/{Id}/samples' + ''' + queryParams = self._validateQueryParameters(queryPars) + resourcePath = '/runs/{Id}/samples' method = 'GET' - resourcePath = resourcePath.replace('{Id}', Id) - headerParams = {} + resourcePath = resourcePath.replace('{Id}', Id) + headerParams = {} return self.__listRequest__(Sample.Sample,resourcePath, method, queryParams, headerParams) - + def getAppResultsByProject(self, Id, queryPars=None, statuses=None): ''' Returns a list of AppResult object associated with the project with Id - + :param Id: The project id :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :param statuses: An (optional) list of AppResult statuses to filter by, eg., 'complete' :returns: a list of AppResult instances ''' - queryParams = self._validateQueryParameters(queryPars) + queryParams = self._validateQueryParameters(queryPars) if statuses is None: - statuses = [] - resourcePath = '/projects/{Id}/appresults' - method = 'GET' - if len(statuses): + statuses = [] + resourcePath = '/projects/{Id}/appresults' + method = 'GET' + if len(statuses): queryParams['Statuses'] = ",".join(statuses) headerParams = {} resourcePath = resourcePath.replace('{Id}',Id) @@ -754,22 +755,22 @@ def getSamplesByProject(self, Id, queryPars=None): def getSampleById(self, Id, queryPars=None): ''' Returns a Sample object - + :param Id: The id of the sample :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a Sample instance ''' - queryParams = self._validateQueryParameters(queryPars) - resourcePath = '/samples/{Id}' + queryParams = self._validateQueryParameters(queryPars) + resourcePath = '/samples/{Id}' method = 'GET' - resourcePath = resourcePath.replace('{Id}', Id) + resourcePath = resourcePath.replace('{Id}', Id) headerParams = {} return self.__singleRequest__(SampleResponse.SampleResponse, resourcePath, method, queryParams, headerParams) - + def getSamplePropertiesById(self, Id, queryPars=None): ''' Returns the Properties of a Sample object - + :param Id: The id of the sample :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a PropertyList instance @@ -785,14 +786,14 @@ def getSamplePropertiesById(self, Id, queryPars=None): def getSampleFilesById(self, Id, queryPars=None): ''' Returns a list of File objects associated with a Sample - + :param Id: A Sample id :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a list of File instances ''' queryParams = self._validateQueryParameters(queryPars) - resourcePath = '/samples/{Id}/files' - method = 'GET' + resourcePath = '/samples/{Id}/files' + method = 'GET' headerParams = {} resourcePath = resourcePath.replace('{Id}',Id) return self.__listRequest__(File.File, @@ -801,43 +802,43 @@ def getSampleFilesById(self, Id, queryPars=None): def getFilesBySample(self, Id, queryPars=None): ''' * Deprecated in favor of getSampleFilesById() * - + Returns a list of File objects associated with a Sample - + :param Id: A Sample id :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a list of File instances ''' - return self.getSampleFilesById(Id, queryPars) - + return self.getSampleFilesById(Id, queryPars) + def getFileById(self, Id, queryPars=None): ''' Returns a file object by Id - + :param Id: The id of the file :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a File instance ''' - queryParams = self._validateQueryParameters(queryPars) - resourcePath = '/files/{Id}' + queryParams = self._validateQueryParameters(queryPars) + resourcePath = '/files/{Id}' method = 'GET' - resourcePath = resourcePath.replace('{Id}', Id) + resourcePath = resourcePath.replace('{Id}', Id) headerParams = {} return self.__singleRequest__(FileResponse.FileResponse, resourcePath, method, queryParams, headerParams) - + def getFilePropertiesById(self, Id, queryPars=None): ''' Returns the Properties of a file object by Id - + :param Id: The id of the file :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a PropertyList instance - ''' - queryParams = self._validateQueryParameters(queryPars) - resourcePath = '/files/{Id}/properties' + ''' + queryParams = self._validateQueryParameters(queryPars) + resourcePath = '/files/{Id}/properties' method = 'GET' - resourcePath = resourcePath.replace('{Id}', Id) + resourcePath = resourcePath.replace('{Id}', Id) headerParams = {} return self.__singleRequest__(PropertiesResponse.PropertiesResponse, resourcePath, method, queryParams, headerParams) @@ -845,7 +846,7 @@ def getFilePropertiesById(self, Id, queryPars=None): def getGenomeById(self, Id, ): ''' Returns an instance of Genome with the specified Id - + :param Id: The genome id :returns: a GenomeV1 instance ''' @@ -862,10 +863,10 @@ def getGenomeById(self, Id, ): def getAvailableGenomes(self, queryPars=None): ''' Returns a list of all available genomes - + :param queryPars: An (optional) object of type QueryParameters for custom sorting and filtering :returns: a list of GenomeV1 instances - ''' + ''' queryParams = self._validateQueryParameters(queryPars) resourcePath = '/genomes' method = 'GET' @@ -876,12 +877,12 @@ def getAvailableGenomes(self, queryPars=None): def getIntervalCoverage(self, Id, Chrom, StartPos, EndPos): ''' Returns metadata about an alignment, including max coverage and cov granularity. - Note that HrefCoverage must be available for the provided BAM file. - + Note that HrefCoverage must be available for the provided BAM file. + :param Id: the Id of a BAM file :param Chrom: chromosome name :param StartPos: get coverage starting at this position - :param EndPos: get coverage up to and including this position; the returned EndPos may be larger than requested due to rounding up to nearest window end coordinate + :param EndPos: get coverage up to and including this position; the returned EndPos may be larger than requested due to rounding up to nearest window end coordinate :returns: a Coverage instance ''' resourcePath = '/coverage/{Id}/{Chrom}' @@ -899,7 +900,7 @@ def getCoverageMetaInfo(self, Id, Chrom): ''' Returns metadata about coverage of a chromosome. Note that HrefCoverage must be available for the provided BAM file - + :param Id: the Id of a Bam file :param Chrom: chromosome name :returns: a CoverageMetaData instance @@ -909,15 +910,15 @@ def getCoverageMetaInfo(self, Id, Chrom): queryParams = {} headerParams = {} resourcePath = resourcePath.replace('{Chrom}', Chrom) - resourcePath = resourcePath.replace('{Id}', Id) + resourcePath = resourcePath.replace('{Id}', Id) return self.__singleRequest__(CoverageMetaResponse.CoverageMetaResponse, resourcePath, method, queryParams, headerParams) def filterVariantSet(self,Id, Chrom, StartPos, EndPos, Format='json', queryPars=None): ''' List the variants in a set of variants. Note the maximum returned records is 1000. - - :param Id: The id of the variant file + + :param Id: The id of the variant file :param Chrom: Chromosome name :param StartPos: The start position of the sequence of interest :param EndPos: The start position of the sequence of interest @@ -927,7 +928,7 @@ def filterVariantSet(self,Id, Chrom, StartPos, EndPos, Format='json', queryPars= ''' queryParams = self._validateQueryParameters(queryPars) resourcePath = '/variantset/{Id}/variants/{Chrom}' - method = 'GET' + method = 'GET' headerParams = {} queryParams['StartPos'] = StartPos queryParams['EndPos'] = EndPos @@ -940,14 +941,14 @@ def filterVariantSet(self,Id, Chrom, StartPos, EndPos, Format='json', queryPars= return self.__listRequest__(Variant.Variant, resourcePath, method, queryParams, headerParams, sort=False) def getVariantMetadata(self, Id, Format='json'): - ''' + ''' Returns the header information of a VCF file. - + :param Id: The Id of the VCF file :param Format: (optional) The return-value format, set to 'json' (default) to return return an object (not actually json format), or 'vcf' (not implemented yet) to return a string in VCF format. :returns: A VariantHeader instance ''' - resourcePath = '/variantset/{Id}' + resourcePath = '/variantset/{Id}' method = 'GET' queryParams = {} headerParams = {} @@ -958,16 +959,16 @@ def getVariantMetadata(self, Id, Format='json'): else: return self.__singleRequest__(VariantsHeaderResponse.VariantsHeaderResponse, resourcePath, method, queryParams, headerParams) - + def createAppResult(self, Id, name, desc, samples=None, appSessionId=None): ''' Create an AppResult object. - + :param Id: The id of the project in which the AppResult is to be added :param name: The name of the AppResult :param desc: A description of the AppResult - :param samples: (Optional) A list of one or more Samples Ids that the AppResult is related to - :param appSessionId: (Optional) If no appSessionId is given, the id used to initialize the BaseSpaceAPI instance will be used. If appSessionId is set equal to an empty string, a new appsession will be created for the appresult object + :param samples: (Optional) A list of one or more Samples Ids that the AppResult is related to + :param appSessionId: (Optional) If no appSessionId is given, the id used to initialize the BaseSpaceAPI instance will be used. If appSessionId is set equal to an empty string, a new appsession will be created for the appresult object :raises Exception: when attempting to create AppResult in an AppSession that has a status other than 'running'. :returns: a newly created AppResult instance ''' @@ -982,12 +983,12 @@ def createAppResult(self, Id, name, desc, samples=None, appSessionId=None): queryParams = {} headerParams = {} postData = {} - + if appSessionId: queryParams['appsessionid'] = appSessionId if appSessionId==None: queryParams['appsessionid'] = self.appSessionId # default case, we use the current appsession - + # add the sample references if len(samples): ref = [] @@ -996,28 +997,28 @@ def createAppResult(self, Id, name, desc, samples=None, appSessionId=None): ref.append(d) postData['References'] = ref # case, an appSession is provided, we need to check if the app is running - if queryParams.has_key('appsessionid'): + if 'appsessionid' in queryParams: session = self.getAppSession(Id=queryParams['appsessionid']) if not session.canWorkOn(): raise Exception('AppSession status must be "running," to create an AppResults. Current status is ' + session.Status) - + postData['Name'] = name postData['Description'] = desc return self.__singleRequest__(AppResultResponse.AppResultResponse, resourcePath, method, queryParams, headerParams, postData=postData) - + def appResultFileUpload(self, Id, localPath, fileName, directory, contentType): ''' Uploads a file associated with an AppResult to BaseSpace and returns the corresponding file object. Small files are uploaded with a single-part upload method, while larger files (> 25 MB) are uploaded with multipart upload. - + :param Id: AppResult id. :param localPath: The local path to the file to be uploaded, including file name. :param fileName: The desired filename in the AppResult folder on the BaseSpace server. :param directory: The directory the file should be placed in on the BaseSpace server. :param contentType: The content-type of the file, eg. 'text/plain' for text files, 'application/octet-stream' for binary files - :returns: a newly created File instance + :returns: a newly created File instance ''' multipart_min_file_size = 25000000 # bytes if os.path.getsize(localPath) > multipart_min_file_size: @@ -1028,11 +1029,11 @@ def appResultFileUpload(self, Id, localPath, fileName, directory, contentType): def createSample(self, Id, name, experimentName, sampleNumber, sampleTitle, readLengths, countRaw, countPF, reference=None, appSessionId=None): ''' Create a Sample object. - + :param Id: The id of the project in which the Sample is to be added :param name: The name of the Sample - :param reference: (Optional) Reference genome that the sample relates to - :param appSessionId: (Optional) If no appSessionId is given, the id used to initialize the BaseSpaceAPI instance will be used. If appSessionId is set equal to an empty string, a new appsession will be created for the sample object + :param reference: (Optional) Reference genome that the sample relates to + :param appSessionId: (Optional) If no appSessionId is given, the id used to initialize the BaseSpaceAPI instance will be used. If appSessionId is set equal to an empty string, a new appsession will be created for the sample object :raises Exception: when attempting to create Sample in an AppSession that has a status other than 'running'. :returns: a newly created Sample instance ''' @@ -1047,18 +1048,18 @@ def createSample(self, Id, name, experimentName, sampleNumber, sampleTitle, read queryParams = {} headerParams = {} postData = {} - + if appSessionId: queryParams['appsessionid'] = appSessionId if appSessionId==None: queryParams['appsessionid'] = self.appSessionId # default case, we use the current appsession - + # case, an appSession is provided, we need to check if the app is running - if queryParams.has_key('appsessionid'): + if 'appsessionid' in queryParams: session = self.getAppSession(Id=queryParams['appsessionid']) if not session.canWorkOn(): raise Exception('AppSession status must be "running," to create a Sample. Current status is ' + session.Status) - + postData['Name'] = name postData['ExperimentName'] = experimentName postData['SampleNumber'] = sampleNumber @@ -1099,14 +1100,14 @@ def __singlepartFileUpload__(self, resourceType, resourceId, localPath, fileName ''' Uploads a file associated with an Endpoint to BaseSpace and returns the corresponding file object. Intended for small files -- reads whole file into memory prior to upload. - + :param resourceType: resource type for the property :param resourceId: identifier for the resource :param localPath: The local path to the file to be uploaded, including file name. :param fileName: The desired filename in the Endpoint folder on the BaseSpace server. :param directory: The directory the file should be placed in on the BaseSpace server. :param contentType: The content-type of the file. - :returns: a newly created File instance + :returns: a newly created File instance ''' if resourceType not in PROPERTY_RESOURCE_TYPES: raise IllegalParameterException(resourceType, PROPERTY_RESOURCE_TYPES) @@ -1116,7 +1117,7 @@ def __singlepartFileUpload__(self, resourceType, resourceId, localPath, fileName resourcePath = resourcePath.replace('{Resource}', resourceType) queryParams = {} queryParams['name'] = fileName - queryParams['directory'] = directory + queryParams['directory'] = directory headerParams = {} headerParams['Content-Type'] = contentType postData = open(localPath).read() @@ -1125,14 +1126,14 @@ def __singlepartFileUpload__(self, resourceType, resourceId, localPath, fileName def __initiateMultipartFileUpload__(self, resourceType, resourceId, fileName, directory, contentType): ''' - Initiates multipart upload of a file to an AppResult in BaseSpace (does not actually upload file). - + Initiates multipart upload of a file to an AppResult in BaseSpace (does not actually upload file). + :param resourceType: resource type for the property :param resourceId: identifier for the resource :param fileName: The desired filename in the AppResult folder on the BaseSpace server. :param directory: The directory the file should be placed in on the BaseSpace server. :param contentType: The content-type of the file, eg. 'text/plain' for text files, 'application/octet-stream' for binary files - :returns: A newly created File instance + :returns: A newly created File instance ''' if resourceType not in PROPERTY_RESOURCE_TYPES: raise IllegalParameterException(resourceType, PROPERTY_RESOURCE_TYPES) @@ -1142,10 +1143,10 @@ def __initiateMultipartFileUpload__(self, resourceType, resourceId, fileName, di resourcePath = resourcePath.replace('{Resource}', resourceType) queryParams = {} queryParams['name'] = fileName - queryParams['directory'] = directory + queryParams['directory'] = directory headerParams = {} headerParams['Content-Type'] = contentType - + queryParams['multipart'] = 'true' postData = None # Set force post as this need to use POST though no data is being streamed @@ -1155,12 +1156,12 @@ def __initiateMultipartFileUpload__(self, resourceType, resourceId, fileName, di def __uploadMultipartUnit__(self, Id, partNumber, md5, data): ''' Uploads file part for multipart upload - - :param Id: file id + + :param Id: file id :param partNumber: the file part to be uploaded :param md5: md5 sum of datastream :param data: the name of the file containing only data to be uploaded - :returns: A dictionary of the server response, with a 'Response' key that contains a dict, which contains an 'ETag' key and value on success. On failure, this method returns None + :returns: A dictionary of the server response, with a 'Response' key that contains a dict, which contains an 'ETag' key and value on success. On failure, this method returns None ''' method = 'PUT' resourcePath = '/files/{Id}/parts/{partNumber}' @@ -1168,12 +1169,12 @@ def __uploadMultipartUnit__(self, Id, partNumber, md5, data): resourcePath = resourcePath.replace('{partNumber}', str(partNumber)) queryParams = {} headerParams = {'Content-MD5':md5.strip()} - return self.apiClient.callAPI(resourcePath, method, queryParams, data, headerParams=headerParams, forcePost=0) + return self.apiClient.callAPI(resourcePath, method, queryParams, data, headerParams=headerParams, forcePost=0) def __finalizeMultipartFileUpload__(self, Id): ''' - Marks a multipart upload file as complete - + Marks a multipart upload file as complete + :param Id: the File Id :returns: a File instance with UploadStatus attribute updated to 'complete' ''' @@ -1182,7 +1183,7 @@ def __finalizeMultipartFileUpload__(self, Id): resourcePath = resourcePath.replace('{Id}', Id) headerParams = {} queryParams = {'uploadstatus':'complete'} - postData = None + postData = None # Set force post as this need to use POST though no data is being streamed return self.__singleRequest__(FileResponse.FileResponse, resourcePath, method, queryParams, headerParams, postData=postData, forcePost=1) @@ -1190,7 +1191,7 @@ def __finalizeMultipartFileUpload__(self, Id): def multipartFileUpload(self, resourceType, resourceId, localPath, fileName, directory, contentType, processCount=10, partSize=25): ''' Method for multi-threaded file-upload for parallel transfer of very large files (currently only runs on unix systems) - + :param resourceType: resource type for the property :param resourceId: identifier for the resource :param localPath: The local path of the file to upload, including file name; local path will not be stored in BaseSpace (use directory argument for this) @@ -1208,7 +1209,7 @@ def multipartFileUpload(self, resourceType, resourceId, localPath, fileName, dir raise UploadPartSizeException("Multipart upload partSize must be >5 MB and <=25 MB") bsFile = self.__initiateMultipartFileUpload__(resourceType, resourceId, fileName, directory, contentType) myMpu = mpu(self, localPath, bsFile, processCount, partSize) - return myMpu.upload() + return myMpu.upload() def multipartFileUploadSample(self, Id, localPath, fileName, directory, contentType, tempDir=None, processCount=10, partSize=25): ''' @@ -1239,21 +1240,21 @@ def multipartFileUploadSample(self, Id, localPath, fileName, directory, contentT def fileDownload(self, Id, localDir, byteRange=None, createBsDir=False): ''' Downloads a BaseSpace file to a local directory, and names the file with the BaseSpace file name. - If the File has a directory in BaseSpace, it will be re-created locally in the provided localDir + If the File has a directory in BaseSpace, it will be re-created locally in the provided localDir (to disable this, set createBsPath=False). - - If the file is large, multi-part download will be used. - + + If the file is large, multi-part download will be used. + Byte-range requests are supported for only small byte ranges (single-part downloads). Byte-range requests are restricted to a single request of 'start' and 'end' bytes, without support for negative or empty values for 'start' or 'end'. - + :param Id: The file id - :param localDir: The local directory to place the file in + :param localDir: The local directory to place the file in :param byteRange: (optional) The byte range of the file to retrieve, provide a 2-element list with start and end byte values :param createBsDir: (optional) create BaseSpace File's directory inside localDir (default: False) :raises ByteRangeException: if the provided byte range is invalid - :returns: a File instance + :returns: a File instance ''' max_retries = 5 multipart_min_file_size = 5000000 # bytes @@ -1266,7 +1267,7 @@ def fileDownload(self, Id, localDir, byteRange=None, createBsDir=False): raise ByteRangeException("Byte range must have smaller byte number first") if rangeSize > multipart_min_file_size: raise ByteRangeException("Byte range %d larger than maximum allowed size %d" % (rangeSize, multipart_min_file_size)) - + bsFile = self.getFileById(Id) if (bsFile.Size < multipart_min_file_size) or (byteRange and (rangeSize < multipart_min_file_size)): # append File's directory to local dir, and create this path if it doesn't exist @@ -1274,7 +1275,7 @@ def fileDownload(self, Id, localDir, byteRange=None, createBsDir=False): if createBsDir: localDest = os.path.join(localDir, os.path.dirname(bsFile.Path)) if not os.path.exists(localDest): - os.makedirs(localDest) + os.makedirs(localDest) attempt = 0 while attempt < max_retries: try: @@ -1286,18 +1287,18 @@ def fileDownload(self, Id, localDir, byteRange=None, createBsDir=False): if attempt == max_retries: raise ServerResponseException("Max retries exceeded") return bsFile - else: + else: return self.multipartFileDownload(Id, localDir, createBsDir=createBsDir) def __downloadFile__(self, Id, localDir, name, byteRange=None, standaloneRangeFile=False, lock=None): #@ReservedAssignment ''' - Downloads a BaseSpace file to a local directory. - Supports byte-range requests; by default will seek() into local file for multipart downloads, + Downloads a BaseSpace file to a local directory. + Supports byte-range requests; by default will seek() into local file for multipart downloads, with option to save only range data in standalone file (no seek()). - - This method is for downloading relatively small files, eg. < 5 MB. - For larger files, use multipart download (which uses this method for file parts). - + + This method is for downloading relatively small files, eg. < 5 MB. + For larger files, use multipart download (which uses this method for file parts). + :param Id: The file id :param localDir: The local directory to place the file in :param name: The name of the local file @@ -1316,24 +1317,24 @@ def __downloadFile__(self, Id, localDir, name, byteRange=None, standaloneRangeFi queryParams = {} headerParams = {} resourcePath = resourcePath.replace('{Id}', Id) - queryParams['redirect'] = 'meta' # we need to add this parameter to get the Amazon link directly - + queryParams['redirect'] = 'meta' # we need to add this parameter to get the Amazon link directly + response = self.apiClient.callAPI(resourcePath, method, queryParams, None, headerParams) - if response['ResponseStatus'].has_key('ErrorCode'): + if 'ErrorCode' in response['ResponseStatus']: raise Exception('BaseSpace error: ' + str(response['ResponseStatus']['ErrorCode']) + ": " + response['ResponseStatus']['Message']) - + # get the Amazon URL, then do the download; for range requests include # size to ensure reading until end of data stream. Create local file if - # it doesn't exist (don't truncate in case other processes from + # it doesn't exist (don't truncate in case other processes from # multipart download also do this) - req = urllib2.Request(response['Response']['HrefContent']) + req = urllib.request.Request(response['Response']['HrefContent']) filename = os.path.join(localDir, name) if not os.path.exists(filename): open(filename, 'a').close() iter_size = 16*1024 # python default if len(byteRange): req.add_header('Range', 'bytes=%s-%s' % (byteRange[0], byteRange[1])) - flo = urllib2.urlopen(req, timeout=self.getTimeout()) # timeout prevents blocking + flo = urllib.request.urlopen(req, timeout=self.getTimeout()) # timeout prevents blocking totRead = 0 with open(filename, 'r+b', 0) as fp: if len(byteRange) and standaloneRangeFile == False: @@ -1360,14 +1361,14 @@ def __downloadFile__(self, Id, localDir, name, byteRange=None, standaloneRangeFi def multipartFileDownload(self, Id, localDir, processCount=10, partSize=25, createBsDir=False, tempDir=""): ''' Method for multi-threaded file-download for parallel transfer of very large files (currently only runs on unix systems) - - :param Id: The ID of the File to download + + :param Id: The ID of the File to download :param localDir: The local path in which to store the downloaded file :param processCount: (optional) The number of processes to be used, default 10 :param partSize: (optional) The size in MB of individual file parts to download, default 25 :param createBsDir: (optional) create BaseSpace File's directory in local_dir, default False :param tempDir: (optional) Set temp directory to use debug mode, which stores downloaded file chunks in individual files, then completes by 'cat'ing chunks into large file - :returns: a File instance + :returns: a File instance ''' myMpd = mpd(self, Id, localDir, processCount, partSize, createBsDir, tempDir) return myMpd.download() @@ -1375,9 +1376,9 @@ def multipartFileDownload(self, Id, localDir, processCount=10, partSize=25, crea def fileUrl(self, Id): ''' ** Deprecated in favor of fileS3metadata() ** - + Returns URL of file (on S3) - + :param Id: The file id :raises Exception: if REST API call to BaseSpace server fails :returns: a URL @@ -1388,17 +1389,17 @@ def fileUrl(self, Id): queryParams = {} headerParams = {} resourcePath = resourcePath.replace('{Id}', Id) - queryParams['redirect'] = 'meta' # we need to add this parameter to get the Amazon link directly - + queryParams['redirect'] = 'meta' # we need to add this parameter to get the Amazon link directly + response = self.apiClient.callAPI(resourcePath, method, queryParams, None, headerParams) - if response['ResponseStatus'].has_key('ErrorCode'): - raise Exception('BaseSpace error: ' + str(response['ResponseStatus']['ErrorCode']) + ": " + response['ResponseStatus']['Message']) + if 'ErrorCode' in response['ResponseStatus']: + raise Exception('BaseSpace error: ' + str(response['ResponseStatus']['ErrorCode']) + ": " + response['ResponseStatus']['Message']) return response['Response']['HrefContent'] def fileS3metadata(self, Id): ''' Returns the S3 url and etag (md5 for small files uploaded as a single part) for a BaseSpace file - + :param Id: The file id :raises Exception: if REST API call to BaseSpace server fails :returns: Dict with s3 url ('url' key) and etag ('etag' key) @@ -1410,20 +1411,20 @@ def fileS3metadata(self, Id): queryParams = {} headerParams = {} resourcePath = resourcePath.replace('{Id}', Id) - queryParams['redirect'] = 'meta' # we need to add this parameter to get the Amazon link directly - + queryParams['redirect'] = 'meta' # we need to add this parameter to get the Amazon link directly + response = self.apiClient.callAPI(resourcePath, method, queryParams,None, headerParams) - if response['ResponseStatus'].has_key('ErrorCode'): + if 'ErrorCode' in response['ResponseStatus']: raise Exception('BaseSpace error: ' + str(response['ResponseStatus']['ErrorCode']) + ": " + response['ResponseStatus']['Message']) - + # record S3 URL ret['url'] = response['Response']['HrefContent'] - + # TODO should use HEAD call here, instead do small GET range request - # GET S3 url and record etag - req = urllib2.Request(response['Response']['HrefContent']) + # GET S3 url and record etag + req = urllib.request.Request(response['Response']['HrefContent']) req.add_header('Range', 'bytes=%s-%s' % (0, 1)) - flo = urllib2.urlopen(req, timeout=self.getTimeout()) # timeout prevents blocking + flo = urllib.request.urlopen(req, timeout=self.getTimeout()) # timeout prevents blocking try: etag = flo.headers['etag'] except KeyError: @@ -1433,12 +1434,12 @@ def fileS3metadata(self, Id): etag = etag[1:-1] ret['etag'] = etag return ret - + def _validateQueryParameters(self, queryPars): ''' Initializes and validates Query Parameter arguments - - :param queryPars: a QueryParameter object + + :param queryPars: a QueryParameter object :return: dictionary of query parameters ''' if queryPars is None: @@ -1461,7 +1462,7 @@ def __dictionaryToProperties__(self, rawProperties, namespace): ''' LEGAL_KEY_TYPES = [str, int, float, bool] propList = [] - for key, value in rawProperties.iteritems(): + for key, value in six.iteritems(rawProperties): if type(value) not in LEGAL_KEY_TYPES: raise IllegalParameterException(type(value), LEGAL_KEY_TYPES) propName = "%s.%s" % (namespace, key) @@ -1506,8 +1507,8 @@ def getResourceProperties(self, resourceType, resourceId): ''' Gets the properties for an arbitrary resource: - https://developer.basespace.illumina.com/docs/content/documentation/rest-api/api-reference#Properties - + https://developer.basespace.illumina.com/docs/content/documentation/rest-api/api-reference#Properties + :param resourceType: resource type for the property Because of this generic treatment of properties (which is fairly new in BaseSpace) @@ -1556,4 +1557,4 @@ def getApplicationById(self, Id): method = 'GET' headerParams = {} queryParams = {} - return self.__singleRequest__(Application.Application, resourcePath, method, queryParams, headerParams) \ No newline at end of file + return self.__singleRequest__(Application.Application, resourcePath, method, queryParams, headerParams) diff --git a/src/BaseSpacePy/api/BillingAPI.py b/src/BaseSpacePy/api/BillingAPI.py index d1af30a..804f8b4 100644 --- a/src/BaseSpacePy/api/BillingAPI.py +++ b/src/BaseSpacePy/api/BillingAPI.py @@ -1,10 +1,12 @@ -import urlparse from BaseSpacePy.api.BaseAPI import BaseAPI from BaseSpacePy.api.BaseSpaceException import * #@UnusedWildImport from BaseSpacePy.model import * #@UnusedWildImport from BaseSpacePy.model.QueryParametersPurchasedProduct import QueryParametersPurchasedProduct as qpp +from six.moves import urllib + + class BillingAPI(BaseAPI): ''' The API class used for all communication with the BaseSpace Billng server @@ -18,7 +20,7 @@ def __init__(self, apiServer, version, appSessionId='', AccessToken=''): ''' self.appSessionId = appSessionId self.version = version - apiServerAndVersion = urlparse.urljoin(apiServer, version) + apiServerAndVersion = urllib.parse.urljoin(apiServer, version) super(BillingAPI, self).__init__(AccessToken, apiServerAndVersion) def createPurchase(self, products, appSessionId=''): diff --git a/src/BaseSpacePy/model/ListResponse.py b/src/BaseSpacePy/model/ListResponse.py index edbf7f1..63d252c 100644 --- a/src/BaseSpacePy/model/ListResponse.py +++ b/src/BaseSpacePy/model/ListResponse.py @@ -1,6 +1,8 @@ import json -from StringIO import StringIO + +from six import StringIO + class ListResponse(object): diff --git a/src/BaseSpacePy/model/MultipartFileTransfer.py b/src/BaseSpacePy/model/MultipartFileTransfer.py index fc5adc3..9058f5f 100644 --- a/src/BaseSpacePy/model/MultipartFileTransfer.py +++ b/src/BaseSpacePy/model/MultipartFileTransfer.py @@ -3,7 +3,6 @@ import os import math import multiprocessing -import Queue import shutil import signal import hashlib @@ -11,31 +10,35 @@ import logging from BaseSpacePy.api.BaseSpaceException import MultiProcessingTaskFailedException +from six.moves import queue +from six.moves import range + + LOGGER = logging.getLogger(__name__) class UploadTask(object): ''' - Uploads a piece of a large local file. - ''' + Uploads a piece of a large local file. + ''' def __init__(self, api, bs_file_id, piece, total_pieces, local_path, total_size, chunk_size): self.api = api self.bs_file_id = bs_file_id # the BaseSpace File Id - self.piece = piece # piece number + self.piece = piece # piece number self.total_pieces = total_pieces # out of total piece count - self.local_path = local_path # the path of the local file to be uploaded, including file name + self.local_path = local_path # the path of the local file to be uploaded, including file name self.total_size = total_size # total file size of upload, for reporting self.chunk_size = chunk_size # chunk size - + # tasks must implement these attributes and execute() self.success = False - self.err_msg = "no error" - + self.err_msg = "no error" + def execute(self, lock): ''' Calculate md5 of file piece and pass to upload method. Lock is not used (but needed since worker sends it for multipart download) - ''' + ''' try: fname = os.path.basename(self.local_path) chunk_data = "" @@ -47,10 +50,10 @@ def execute(self, lock): res = self.api.__uploadMultipartUnit__(self.bs_file_id,self.piece+1,self.md5,chunk_data) except Exception as e: self.success = False - self.err_msg = str(e) + self.err_msg = str(e) else: # ETag contains hex encoded MD5 of part data on success - if res and res['Response'].has_key('ETag'): + if res and 'ETag' in res['Response']: self.success = True else: self.success = False @@ -60,7 +63,7 @@ def execute(self, lock): self.success = False self.err_msg = str(e) return self - + def __str__(self): return 'File piece %d of %d, total %s' % (self.piece+1, self.total_pieces, Utils.readable_bytes(self.total_size)) @@ -69,7 +72,7 @@ class DownloadTask(object): ''' Downloads a piece of a large remote file. When temp_dir is set (debug mode), downloads to filename with piece number appended (i.e. temp file). - ''' + ''' def __init__(self, api, bs_file_id, file_name, local_dir, piece, total_pieces, part_size, total_size, temp_dir=None): self.api = api # BaseSpace api object self.bs_file_id = bs_file_id # the Id of the File in BaseSpace @@ -78,13 +81,13 @@ def __init__(self, api, bs_file_id, file_name, local_dir, piece, total_pieces, p self.total_pieces = total_pieces # total pieces being downloaded (for reporting only) self.part_size = part_size # the size in bytes (not MB) of each piece (except last piece) self.total_size = total_size # the total size of the file in bytes - self.local_dir = local_dir # the path in which to store the downloaded file - self.temp_dir = temp_dir # optional: set temp_dir for debug mode, which writes downloaded chunks to individual temp files - + self.local_dir = local_dir # the path in which to store the downloaded file + self.temp_dir = temp_dir # optional: set temp_dir for debug mode, which writes downloaded chunks to individual temp files + # tasks must implement these attributes and execute() self.success = False - self.err_msg = "no error" - + self.err_msg = "no error" + def execute(self, lock): ''' Download a piece of the target file, first calculating start/end bytes for piece. @@ -104,147 +107,147 @@ def execute(self, lock): startbyte = (self.piece - 1) * self.part_size endbyte = (self.piece * self.part_size) - 1 if endbyte > self.total_size: - endbyte = self.total_size - 1 - try: + endbyte = self.total_size - 1 + try: #self.api.__downloadFile__(self.bs_file_id, self.local_dir, transFile, [startbyte, endbyte], standaloneRangeFile, lock) - self.api.__downloadFile__(self.bs_file_id, local_dir, local_name, [startbyte, endbyte], standaloneRangeFile, lock) + self.api.__downloadFile__(self.bs_file_id, local_dir, local_name, [startbyte, endbyte], standaloneRangeFile, lock) except Exception as e: self.success = False - self.err_msg = str(e) - else: + self.err_msg = str(e) + else: self.success = True # capture exception, since unpickleable exceptions may block except Exception as e: self.success = False self.err_msg = str(e) return self - - def __str__(self): + + def __str__(self): return 'File piece %d of %d, piece size %s of total %s' % (self.piece, self.total_pieces, Utils.readable_bytes(self.part_size), Utils.readable_bytes(self.total_size)) - + class Consumer(multiprocessing.Process): ''' Multi-processing worker that executes tasks from task queue with retry On failure after retries, alerts all workers to halt ''' - - def __init__(self, task_queue, result_queue, halt_event, lock): + + def __init__(self, task_queue, result_queue, halt_event, lock): multiprocessing.Process.__init__(self) self.task_queue = task_queue - self.result_queue = result_queue + self.result_queue = result_queue self.halt = halt_event - self.lock = lock - + self.lock = lock + self.get_task_timeout = 5 # secs self.retry_wait = 1 # sec self.retries = 20 - + def run(self): ''' - Executes tasks from the task queue until poison pill is reached, halt + Executes tasks from the task queue until poison pill is reached, halt signal is found, or something went wrong such as a timeout when getting - new tasks. - + new tasks. + For download tasks, use lock to ensure sole access (among worker processes) to downloaded file. - + Retries failed tasks, and add task results to result_queue. When a task fails for all retries, set halt signal to alert other workers and purge task queue or remaining tasks (to unblock join() in parent process) - + Turn off SIGINT (Ctrl C), handle in parent process ''' signal.signal(signal.SIGINT, signal.SIG_IGN) - while True: + while True: try: next_task = self.task_queue.get(True, self.get_task_timeout) # block until timeout - except Queue.Empty: + except queue.Empty: LOGGER.debug('Worker %s exiting, getting task from task queue timed out and/or is empty' % self.name) - break - if next_task is None: + break + if next_task is None: LOGGER.debug('Worker %s exiting, found final task' % self.name) self.task_queue.task_done() - break - else: + break + else: # attempt to run tasks, with retry LOGGER.debug('Worker %s processing task: %s' % (self.name, str(next_task))) LOGGER.info('%s' % str(next_task)) - for i in xrange(1, self.retries + 1): + for i in range(1, self.retries + 1): if self.halt.is_set(): LOGGER.debug('Worker %s exiting, found halt signal' % self.name) self.task_queue.task_done() self.purge_task_queue() - return - answer = next_task.execute(self.lock) # acquired lock will block other workers + return + answer = next_task.execute(self.lock) # acquired lock will block other workers if answer.success == True: - self.task_queue.task_done() + self.task_queue.task_done() self.result_queue.put(True) break else: LOGGER.debug("Worker %s retrying task %s after failure, retry attempt %d, with error msg: %s" % (self.name, str(next_task), i, answer.err_msg)) - time.sleep(self.retry_wait) + time.sleep(self.retry_wait) if not answer.success == True: LOGGER.debug("Worker %s exiting, too many failures with retry for worker %s" % (self.name, str(self))) - LOGGER.warning("Task failed after too many retries") - self.task_queue.task_done() + LOGGER.warning("Task failed after too many retries") + self.task_queue.task_done() self.result_queue.put(False) - self.purge_task_queue() # purge task queue in case there's only one worker + self.purge_task_queue() # purge task queue in case there's only one worker self.halt.set() - break + break return - + def purge_task_queue(self): ''' Purge all remaining tasks from task queue. This will also remove poison pills (final tasks), so run() must handle an empty queue (by using a timeout with task_queue.get() ). - ''' + ''' LOGGER.debug("Purging task queue") while 1: try: - self.task_queue.get(False) - except Queue.Empty: + self.task_queue.get(False) + except queue.Empty: break else: self.task_queue.task_done() - + class Executor(object): ''' Multi-processing task manager, with callback to finalize once workers are completed. Task queue contains tasks, with poison pill for each worker. Result queue contains True/False results for task success/failure. Halt event will tell workers to halt themselves. - - For downloads, lock is to ensure that only one worker writes to a local downloaded file at a time. + + For downloads, lock is to ensure that only one worker writes to a local downloaded file at a time. ''' - def __init__(self): + def __init__(self): self.tasks = multiprocessing.JoinableQueue() self.result_queue = multiprocessing.Queue() self.halt_event = multiprocessing.Event() self.lock = multiprocessing.Lock() - + def add_task(self, task): ''' Add task to task queue ''' - self.tasks.put(task) - + self.tasks.put(task) + def add_workers(self, num_workers): ''' Added workers to internal list of workers, adding a poison pill for each to the task queue ''' - self.consumers = [ Consumer(self.tasks, self.result_queue, self.halt_event, self.lock) for i in xrange(num_workers) ] + self.consumers = [ Consumer(self.tasks, self.result_queue, self.halt_event, self.lock) for i in range(num_workers) ] for c in self.consumers: self.tasks.put(None) def start_workers(self, finalize_callback): ''' Start workers, wait until workers finish, then call finalize callback if all went well - ''' + ''' # TODO add failure callback for cleanup? for w in self.consumers: - w.start() - LOGGER.debug("Workers started") + w.start() + LOGGER.debug("Workers started") try: self.tasks.join() except (KeyboardInterrupt, SystemExit): @@ -252,39 +255,39 @@ def start_workers(self, finalize_callback): self.result_queue.put(False) self.halt_event.set() self.tasks.join() # wait for workers to finish current work then exit from response to halt signal - else: - LOGGER.debug("Workers finished - task queue joined") + else: + LOGGER.debug("Workers finished - task queue joined") finalize = True while 1: try: - success = self.result_queue.get(False) # non-blocking - except Queue.Empty: + success = self.result_queue.get(False) # non-blocking + except queue.Empty: break - else: - if success == False: + else: + if success == False: LOGGER.debug("Found a failed or cancelled task -- won't call finalize callback") - finalize = False - if finalize == True: + finalize = False + if finalize == True: finalize_callback() - else: - raise MultiProcessingTaskFailedException("Multiprocessing task did not complete successfully") + else: + raise MultiProcessingTaskFailedException("Multiprocessing task did not complete successfully") class MultipartUpload(object): ''' - Uploads a (large) file by uploading file parts in separate processes. + Uploads a (large) file by uploading file parts in separate processes. ''' def __init__(self, api, local_path, bs_file, process_count, part_size, logger=None): ''' Create a multipart upload object - - :param api: the BaseSpace API object + + :param api: the BaseSpace API object :param local_path: the path of the local file, including file name - :param bs_file: the File object of the newly created BaseSpace File to upload + :param bs_file: the File object of the newly created BaseSpace File to upload :param process_count: the number of process to use for uploading - :param part_size: in MB, the size of each uploaded part + :param part_size: in MB, the size of each uploaded part ''' - self.api = api - self.local_path = local_path + self.api = api + self.local_path = local_path self.remote_file = bs_file self.process_count = process_count self.part_size = part_size @@ -297,13 +300,13 @@ def upload(self): BaseSpace that has updated (completed) attributes. ''' self._setup() - self._start_workers() - return self.api.getFileById(self.remote_file.Id) - - def _setup(self): + self._start_workers() + return self.api.getFileById(self.remote_file.Id) + + def _setup(self): + ''' + Determine number of file pieces to upload, add upload tasks to work queue ''' - Determine number of file pieces to upload, add upload tasks to work queue - ''' from math import ceil total_size = os.path.getsize(self.local_path) # round up to get a number of chunks that will be enough for the whole file @@ -323,12 +326,12 @@ def _setup(self): # err_msg = "Splitting local file failed: %s" % self.local_path # raise MultiProcessingTaskFailedException(err_msg) - self.exe = Executor() - for i in xrange(self.start_chunk, fileCount): + self.exe = Executor() + for i in range(self.start_chunk, fileCount): t = UploadTask(self.api, self.remote_file.Id, i, fileCount, self.local_path, total_size, chunk_size) - self.exe.add_task(t) + self.exe.add_task(t) self.exe.add_workers(self.process_count) - self.task_total = fileCount - self.start_chunk + 1 + self.task_total = fileCount - self.start_chunk + 1 LOGGER.info("Total File Size %s" % Utils.readable_bytes(total_size)) LOGGER.info("Using File Part Size %d MB" % self.part_size) @@ -339,15 +342,15 @@ def _setup(self): def _start_workers(self): ''' Start upload workers, register finalize callback method - ''' - finalize_callback = self._finalize_upload # lambda: None + ''' + finalize_callback = self._finalize_upload # lambda: None self.exe.start_workers(finalize_callback) - + def _finalize_upload(self): ''' Set file upload status as complete in BaseSpace ''' - LOGGER.debug("Marking uploaded file status as complete") + LOGGER.debug("Marking uploaded file status as complete") self.api.__finalizeMultipartFileUpload__(self.remote_file.Id) class MultipartDownload(object): @@ -359,105 +362,105 @@ class MultipartDownload(object): def __init__(self, api, file_id, local_dir, process_count, part_size, create_bs_dir, temp_dir=""): ''' Create a multipart download object - + :param api: the BaseSpace API object :param file_id: the BaseSpace File Id of the file to download :param local_dir: the local directory in which to store the downloaded file :param process_count: the number of process to use for downloading - :param part_size: in MB, the size of each file part to download + :param part_size: in MB, the size of each file part to download :param create_bs_dir: when True, create BaseSpace File's directory in local_dir; when False, ignore Bs directory - :param temp_dir: (optional) temp directory for debug mode + :param temp_dir: (optional) temp directory for debug mode ''' - self.api = api - self.file_id = file_id - self.local_dir = local_dir - self.process_count = process_count - self.part_size = part_size + self.api = api + self.file_id = file_id + self.local_dir = local_dir + self.process_count = process_count + self.part_size = part_size self.temp_dir = temp_dir - self.create_bs_dir = create_bs_dir + self.create_bs_dir = create_bs_dir - self.start_chunk = 1 + self.start_chunk = 1 self.partial_file_ext = ".partial" - + def download(self): ''' Start the download ''' self._setup() self._start_workers() - return self.bs_file - + return self.bs_file + def _setup(self): ''' Determine number of file pieces to download, determine full local path in which to download file, add download tasks to work queue. - - While download is in progress, name the file with a 'partial' extension + + While download is in progress, name the file with a 'partial' extension ''' self.bs_file = self.api.getFileById(self.file_id) self.file_name = self.bs_file.Name total_bytes = self.bs_file.Size part_size_bytes = self.part_size * (1024**2) self.file_count = int(math.ceil(total_bytes/part_size_bytes)) + 1 - + file_name = self.file_name if not self.temp_dir: file_name = self.file_name + self.partial_file_ext self.full_local_dir = self.local_dir self.full_temp_dir = self.temp_dir - if self.create_bs_dir: - self.full_local_dir = os.path.join(self.local_dir, os.path.dirname(self.bs_file.Path)) + if self.create_bs_dir: + self.full_local_dir = os.path.join(self.local_dir, os.path.dirname(self.bs_file.Path)) if not os.path.exists(self.full_local_dir): os.makedirs(self.full_local_dir) if self.temp_dir: self.full_temp_dir = os.path.join(self.temp_dir, os.path.dirname(self.bs_file.Path)) if not os.path.exists(self.full_temp_dir): os.makedirs(self.full_temp_dir) - - self.exe = Executor() - for i in xrange(self.start_chunk, self.file_count+1): - t = DownloadTask(self.api, self.file_id, file_name, self.full_local_dir, + + self.exe = Executor() + for i in range(self.start_chunk, self.file_count+1): + t = DownloadTask(self.api, self.file_id, file_name, self.full_local_dir, i, self.file_count, part_size_bytes, total_bytes, self.full_temp_dir) - self.exe.add_task(t) - self.exe.add_workers(self.process_count) - self.task_total = self.file_count - self.start_chunk + 1 - + self.exe.add_task(t) + self.exe.add_workers(self.process_count) + self.task_total = self.file_count - self.start_chunk + 1 + LOGGER.debug("Total File Size %s" % Utils.readable_bytes(total_bytes)) LOGGER.debug("Using File Part Size %s MB" % str(self.part_size)) LOGGER.debug("Processes %d" % self.process_count) LOGGER.debug("File Chunk Count %d" % self.file_count) LOGGER.debug("Start Chunk %d" % self.start_chunk) - + def _start_workers(self): ''' Start download workers, register finalize callback method - ''' + ''' if self.temp_dir: finalize_callback = self._combine_file_chunks else: - finalize_callback = self._rename_final_file # lambda: None - self.exe.start_workers(finalize_callback) - + finalize_callback = self._rename_final_file # lambda: None + self.exe.start_workers(finalize_callback) + def _rename_final_file(self): ''' Remove the 'partial' extension from the downloaded file ''' final_file = os.path.join(self.full_local_dir, self.file_name) partial_file = final_file + self.partial_file_ext - os.rename(partial_file, final_file) - + os.rename(partial_file, final_file) + def _combine_file_chunks(self): ''' Assembles download files chunks into single large file, then cleanup by deleting file chunks - ''' - LOGGER.debug("Assembling downloaded file parts into single file") - part_files = [os.path.join(self.full_temp_dir, self.file_name + '.' + str(i)) for i in xrange(self.start_chunk, self.file_count+1)] + ''' + LOGGER.debug("Assembling downloaded file parts into single file") + part_files = [os.path.join(self.full_temp_dir, self.file_name + '.' + str(i)) for i in range(self.start_chunk, self.file_count+1)] with open(os.path.join(self.full_local_dir, self.file_name), 'w+b') as whole_file: - for part_file in part_files: - shutil.copyfileobj(open(part_file, 'r+b'), whole_file) + for part_file in part_files: + shutil.copyfileobj(open(part_file, 'r+b'), whole_file) for part_file in part_files: - os.remove(part_file) + os.remove(part_file) class Utils(object): ''' @@ -475,7 +478,7 @@ def md5_for_file(f, block_size=1024*1024): break md5.update(data) return md5.hexdigest() - + @staticmethod def readable_bytes(size, precision=2): """ @@ -487,4 +490,3 @@ def readable_bytes(size, precision=2): suffixIndex += 1 # increment the index of the suffix size = size / 1024.0 # apply the division return "%.*f %s" % (precision, size, suffixes[suffixIndex]) - \ No newline at end of file diff --git a/src/BaseSpacePy/model/QueryParameters.py b/src/BaseSpacePy/model/QueryParameters.py index f5570e2..0118f9b 100644 --- a/src/BaseSpacePy/model/QueryParameters.py +++ b/src/BaseSpacePy/model/QueryParameters.py @@ -1,4 +1,4 @@ - +import six from BaseSpacePy.api.BaseSpaceException import UndefinedParameterException, UnknownParameterException, IllegalParameterException, QueryParameterException legal = {'Statuses': [], @@ -8,66 +8,66 @@ #'Extensions': ['bam', 'vcf'], 'Offset': [], 'Limit': [], - 'SortDir': ['Asc', 'Desc'], - 'Name': [], - 'StartPos':[], - 'EndPos':[], + 'SortDir': ['Asc', 'Desc'], + 'Name': [], + 'StartPos':[], + 'EndPos':[], 'Format':[], 'include':[], 'propertyFilters':[], 'userCreatedBy':[] - #'Format': ['txt', 'json', 'vcf'], + #'Format': ['txt', 'json', 'vcf'], } class QueryParameters(object): ''' The QueryParameters class can be passed as an optional argument - for sorting/filtering of list-responses (such as lists of samples, AppResults, variants, etc.) + for sorting/filtering of list-responses (such as lists of samples, AppResults, variants, etc.) ''' def __init__(self, pars=None, required=None): ''' :param pars: (optional) a dictionary of query parameters, default None :param required: (optional) a list of required query parameter names, default None - + :raises QueryParameterException: when non-dictionary argument for 'pars' is passed ''' if pars is None: pars = {} if required is None: - required = [] + required = [] self.passed = {} try: - for k in pars.keys(): + for k in six.viewkeys(pars): self.passed[k] = pars[k] except AttributeError: raise QueryParameterException("The 'pars' argument to QueryParameters must be a dictionary") self.required = required - + def __str__(self): return str(self.passed) - + def __repr__(self): return str(self) - + def getParameterDict(self): return self.passed - + def validate(self): ''' Validates that query parameter keys and values are properly formed: - required keys are present, and keys and values are within the set of + required keys are present, and keys and values are within the set of known acceptable keys/values. - + :raises UndefinedParameterException: when a required parameter is not present :raises UnknownParameterException: when a parameter name is not present in the list of acceptable parameters names :raises IllegalParameterException: when a parameter value (with a valid name) is not present in the list of acceptable parameters values :returns: None ''' for p in self.required: - if not self.passed.has_key(p): + if not p in self.passed: raise UndefinedParameterException(p) - for p in self.passed.keys(): - if not legal.has_key(p): + for p in six.viewkeys(self.passed): + if not p in legal: raise UnknownParameterException(p) - if len(legal[p])>0 and (not self.passed[p] in legal[p]): + if len(legal[p])>0 and (not self.passed[p] in legal[p]): raise IllegalParameterException(p,legal[p]) diff --git a/src/BaseSpacePy/model/QueryParametersPurchasedProduct.py b/src/BaseSpacePy/model/QueryParametersPurchasedProduct.py index 44279d3..3306a2b 100644 --- a/src/BaseSpacePy/model/QueryParametersPurchasedProduct.py +++ b/src/BaseSpacePy/model/QueryParametersPurchasedProduct.py @@ -1,4 +1,4 @@ - +import six from BaseSpacePy.api.BaseSpaceException import UndefinedParameterException,UnknownParameterException,IllegalParameterException legal = { 'Tags':[], 'ProductIds':[] } @@ -11,20 +11,20 @@ def __init__(self, pars=None): if pars is None: pars = {} self.passed = {} - for k in pars.keys(): + for k in six.viewkeys(pars): self.passed[k] = pars[k] self.validate() - + def __str__(self): return str(self.passed) - + def __repr__(self): return str(self) - + def getParameterDict(self): return self.passed - + def validate(self): - for p in self.passed.keys(): - if not legal.has_key(p): + for p in six.viewkeys(self.passed): + if not p in legal: raise UnknownParameterException(p) diff --git a/src/setup.py b/src/setup.py index 0458d32..af8ca22 100755 --- a/src/setup.py +++ b/src/setup.py @@ -5,7 +5,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,7 +24,7 @@ setup(name='basespace-python-sdk', description='A Python SDK for connecting to Illumina BaseSpace data', author='Illumina', - version='0.4', + version='0.5', long_description=""" BaseSpacePy is a Python based SDK to be used in the development of Apps and scripts for working with Illumina's BaseSpace cloud-computing solution for next-gen sequencing data analysis. @@ -33,7 +33,7 @@ author_email='techsupport@illumina.com', packages=['BaseSpacePy.api','BaseSpacePy.model','BaseSpacePy'], package_dir={'BaseSpacePy' : os.path.join(os.path.dirname(__file__),'BaseSpacePy')}, - install_requires=['python-dateutil','requests'], + install_requires=['python-dateutil','requests','six'], zip_safe=False, ) @@ -42,5 +42,4 @@ #try: # import dateutil #except: -# print "WARNING - please install required package 'python-dateutil'" - +# print("WARNING - please install required package 'python-dateutil'") diff --git a/test/launch_helpers_tests.py b/test/launch_helpers_tests.py index f0c47b0..2e92c5e 100644 --- a/test/launch_helpers_tests.py +++ b/test/launch_helpers_tests.py @@ -36,10 +36,10 @@ 'sample-id', ] app_defaults = { - 'AnnotationSource': u'RefSeq', - 'genome-id': u'Human', - 'GQX-id': u'30', - 'StrandBias-id': u'10', + 'AnnotationSource': 'RefSeq', + 'genome-id': 'Human', + 'GQX-id': '30', + 'StrandBias-id': '10', 'FlagPCRDuplicates-id': [] } diff --git a/test/test_models.py b/test/test_models.py index aa79ff9..c323fdd 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -7,79 +7,82 @@ from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI from BaseSpacePy.model.QueryParameters import QueryParameters as qp +import six + + class TestSDK(object): ''' Compares objects from BaseSpace REST API to SDK objects, including pickled objects ''' - def __init__(self): - + def __init__(self): + self.qp = {} self.rest_method = 'GET' self.postData = None self.headerParams=None self.list_request = False - + # TODO change to unit_tests, but need to add projects/run to account? - self.myAPI = BaseSpaceAPI(profile="ps_native_hoth") - self.api = APIClient(self.myAPI.getAccessToken(), self.myAPI.apiServer) + self.myAPI = BaseSpaceAPI(profile="ps_native_hoth") + self.api = APIClient(self.myAPI.getAccessToken(), self.myAPI.apiServer) def compare_dict_to_obj(self, rest_dict, p_obj): - """ + """ Compare a dictionary from a REST API response and a SDK object for identity. """ - for r_key, r_val in rest_dict.iteritems(): + for r_key, r_val in six.iteritems(rest_dict): # confirm that the key from REST api exists in stored object try: - p_val = getattr(p_obj, r_key) + p_val = getattr(p_obj, r_key) except AttributeError: - print "REST API attribute '" + r_key + "' doesn't exist in object" + print("REST API attribute '" + r_key + "' doesn't exist in object") else: - self.classify_rest_item(r_val, p_val, r_key) + self.classify_rest_item(r_val, p_val, r_key) def compare_list_to_obj(self, rest_list, p_obj, r_key): - """ + """ Compare a list from a REST API response and an SDK object for identity. - """ + """ if type(p_obj) != list: - print "Attribute '" + r_key + "' is a list in the REST API but not in the object" + print("Attribute '" + r_key + "' is a list in the REST API but not in the object") elif len(p_obj) != len(rest_list): - print "Attribute '" + r_key + "' has different list length between REST API and object" + print("Attribute '" + r_key + "' has different list length between REST API and object") else: for r_val, p_val in map(None, rest_list, p_obj): self.classify_rest_item(r_val, p_val, r_key) - + def compare_builtin_to_obj(self, rest_val, p_obj, r_key): - """ + """ Compare a built-in type from a REST API response and an SDK object for identity. - """ + """ # convert unicode to ascii for comparisons - if isinstance(rest_val, unicode): + if isinstance(rest_val, six.text_type): rest_val = rest_val.encode('ascii','ignore') # don't compare values for datetimes if r_key in ['DateCreated', 'DateModified', 'DateUploadCompleted', 'DateUploadStarted']: pass - elif rest_val != p_obj: - print "REST API attribute '" + r_key + "' has value '" + str(rest_val) + "' doesn't match object value '" + str(p_obj) + "'" + elif rest_val != p_obj: + print("REST API attribute '" + r_key + "' has value '" + str(rest_val) + "' doesn't match object value '" + str(p_obj) + "'") def classify_rest_item(self, r_val, p_val, r_key): """ Determine the input REST item's type and call method to compare to input object - """ - if type(r_val) in [ int, str, bool, float, unicode]: + """ + if type(r_val) in [ int, str, bool, float, six.text_type]: self.compare_builtin_to_obj(r_val, p_val, r_key) - elif type(r_val) == dict: + elif type(r_val) == dict: self.compare_dict_to_obj(r_val, p_val) - elif type(r_val) == list: + elif type(r_val) == list: self.compare_list_to_obj(r_val, p_val, r_key) else: - print "REST API attribute'" + r_key + "' has an unrecognized attribute type" - + print("REST API attribute'" + r_key + "' has an unrecognized attribute type") + def test_rest_vs_sdk(self): """ Compares REST API response and python SDK object for identify, for an API method - """ + """ sdk_obj = self.call_sdk() - rest_obj = self.call_rest_api() + rest_obj = self.call_rest_api() # TODO passing Response here, SDK doesn't currently capture other items at this level (e.g. Notifications) if self.list_request: self.compare_list_to_obj(rest_obj['Response']['Items'], sdk_obj, "BASE") @@ -96,18 +99,18 @@ def create_pickle_from_sdk(self, pickle_path): """ Stores a pickled object in the provided path (include file name) for the object returned for this SDK method """ - sdk_obj = self.call_sdk() + sdk_obj = self.call_sdk() with open(pickle_path, 'w') as f: Pickle.dump(sdk_obj, f) - + def get_pickle(self, pickle_path): """ Retrieves a pickled object from the provided path (include file name), for this API test - """ + """ with open(pickle_path, 'r') as f: sdk_obj = Pickle.load(f) - return sdk_obj - + return sdk_obj + def test_rest_vs_pickle(self, pickle_path): """ Compares REST API response and a stored object for identify, for an API method @@ -118,46 +121,46 @@ def test_rest_vs_pickle(self, pickle_path): class GetAppSessionById(TestSDK): - def __init__(self, bsid): + def __init__(self, bsid): super(GetAppSessionById, self).__init__() self.rest_path = '/appsessions/' + bsid - + self.appsession_id = bsid def call_sdk(self): return self.myAPI.getAppSessionById(self.appsession_id) class GetAppSessionPropertiesById(TestSDK): - + def __init__(self, ssn_id, query_p={}): super(GetAppSessionPropertiesById, self).__init__() self.rest_path = '/appsessions/' + ssn_id + '/properties' - + self.appsession_id = ssn_id self.qp = query_p def call_sdk(self): - return self.myAPI.getAppSessionPropertiesById(self.appsession_id, qp(self.qp)) + return self.myAPI.getAppSessionPropertiesById(self.appsession_id, qp(self.qp)) class GetAppSessionPropertyByName(TestSDK): def __init__(self, ssn_id, prop_name, query_p={}): super(GetAppSessionPropertyByName, self).__init__() self.rest_path = '/appsessions/' + ssn_id + '/properties/' + prop_name + '/items' - + self.appsession_id = ssn_id self.property_name = prop_name self.qp = query_p def call_sdk(self): - return self.myAPI.getAppSessionPropertyByName(self.appsession_id, qp(self.qp), self.property_name) + return self.myAPI.getAppSessionPropertyByName(self.appsession_id, qp(self.qp), self.property_name) class GetRunById(TestSDK): - def __init__(self, bsid, query_p={}): + def __init__(self, bsid, query_p={}): super(GetRunById, self).__init__() self.rest_path = '/runs/' + bsid - + self.run_id = bsid self.qp = query_p @@ -166,10 +169,10 @@ def call_sdk(self): class GetRunPropertiesById(TestSDK): - def __init__(self, ssn_id, query_p={}): + def __init__(self, ssn_id, query_p={}): super(GetRunPropertiesById, self).__init__() self.rest_path = '/runs/' + ssn_id + '/properties' - + self.run_id = ssn_id self.qp = query_p @@ -178,10 +181,10 @@ def call_sdk(self): class GetProjectById(TestSDK): - def __init__(self, bsid, query_p={}): + def __init__(self, bsid, query_p={}): super(GetProjectById, self).__init__() self.rest_path = '/projects/' + bsid - + self.project_id = bsid self.qp = query_p @@ -190,10 +193,10 @@ def call_sdk(self): class GetProjectPropertiesById(TestSDK): - def __init__(self, bsid, query_p={}): + def __init__(self, bsid, query_p={}): super(GetProjectPropertiesById, self).__init__() self.rest_path = '/projects/' + bsid + '/properties' - + self.project_id = bsid self.qp = query_p @@ -202,10 +205,10 @@ def call_sdk(self): class GetSampleById(TestSDK): - def __init__(self, bsid, query_p={}): + def __init__(self, bsid, query_p={}): super(GetSampleById, self).__init__() self.rest_path = '/samples/' + bsid - + self.sample_id = bsid self.qp = query_p @@ -214,10 +217,10 @@ def call_sdk(self): class GetSamplePropertiesById(TestSDK): - def __init__(self, bsid, query_p={}): + def __init__(self, bsid, query_p={}): super(GetSamplePropertiesById, self).__init__() self.rest_path = '/samples/' + bsid + '/properties' - + self.sample_id = bsid self.qp = query_p @@ -226,10 +229,10 @@ def call_sdk(self): class GetAppResultById(TestSDK): - def __init__(self, bsid, query_p={}): + def __init__(self, bsid, query_p={}): super(GetAppResultById, self).__init__() self.rest_path = '/appresults/' + bsid - + self.appresult_id = bsid self.qp = query_p @@ -238,10 +241,10 @@ def call_sdk(self): class GetAppResultPropertiesById(TestSDK): - def __init__(self, bsid, query_p={}): + def __init__(self, bsid, query_p={}): super(GetAppResultPropertiesById, self).__init__() self.rest_path = '/appresults/' + bsid + '/properties' - + self.appresult_id = bsid self.qp = query_p @@ -250,10 +253,10 @@ def call_sdk(self): class GetFileById(TestSDK): - def __init__(self, bsid, query_p={}): + def __init__(self, bsid, query_p={}): super(GetFileById, self).__init__() self.rest_path = '/files/' + bsid - + self.file_id = bsid self.qp = query_p @@ -262,10 +265,10 @@ def call_sdk(self): class GetFilePropertiesById(TestSDK): - def __init__(self, bsid, query_p={}): + def __init__(self, bsid, query_p={}): super(GetFilePropertiesById, self).__init__() self.rest_path = '/files/' + bsid + '/properties' - + self.file_id = bsid self.qp = query_p @@ -273,26 +276,26 @@ def call_sdk(self): return self.myAPI.getFilePropertiesById(self.file_id, qp(self.qp)) class FilterVariantSet(TestSDK): - - def __init__(self, bsid, chrom, start_pos, end_pos, format ,query_p=None): + + def __init__(self, bsid, chrom, start_pos, end_pos, format ,query_p=None): super(FilterVariantSet, self).__init__() - self.rest_path = '/variantset/' + bsid + '/variants/' + chrom + self.rest_path = '/variantset/' + bsid + '/variants/' + chrom if query_p is None: query_p = {} - + self.file_id = bsid self.chrom = chrom self.start_pos = start_pos self.end_pos = end_pos - self.format = format + self.format = format self.qp = copy.deepcopy(query_p) self.qp['StartPos'] = start_pos self.qp['EndPos'] = end_pos self.qp['Format'] = format self.list_request = True - + def call_sdk(self): - return self.myAPI.filterVariantSet(self.file_id, self.chrom, self.start_pos, self.end_pos, self.format, qp(self.qp)) + return self.myAPI.filterVariantSet(self.file_id, self.chrom, self.start_pos, self.end_pos, self.format, qp(self.qp)) class TestSuite(object): @@ -307,73 +310,73 @@ def add_tests(self): cfg = self.cfg try: self.tests.append((FilterVariantSet(cfg['vcf_id'], cfg['vcf_chr'], cfg['vcf_start'], cfg['vcf_end'], cfg['vcf_format'], cfg['query_p']), "with query parameter")) - except AttributeError: - print "Skipping test FilterVariantSet -- missing input parameter" + except AttributeError: + print("Skipping test FilterVariantSet -- missing input parameter") try: self.tests.append((GetAppSessionById(cfg['ssn_id']), "test")) except AttributeError: - print "Skipping test GetAppSessionById -- missing input parameter" + print("Skipping test GetAppSessionById -- missing input parameter") try: self.tests.append((GetAppSessionPropertiesById(cfg['ssn_id'], cfg['query_p']), "with query parameter")) except AttributeError: - print "Skipping test GetAppSessionPropertiesById -- missing input parameter" - try: - for key, value in cfg['multivalue_property_names'].iteritems(): + print("Skipping test GetAppSessionPropertiesById -- missing input parameter") + try: + for key, value in six.iteritems(cfg['multivalue_property_names']): self.tests.append((GetAppSessionPropertyByName(cfg['ssn_id'], value, cfg['query_p']), key)) except AttributeError: - print "Skipping test GetAppSessionPropertiesByName -- missing input parameter" + print("Skipping test GetAppSessionPropertiesByName -- missing input parameter") try: self.tests.append((GetRunById(cfg['run_id'], cfg['query_p']), "with query parameter")) except AttributeError: - print "Skipping test GetRunById -- missing input parameter" + print("Skipping test GetRunById -- missing input parameter") try: self.tests.append((GetRunPropertiesById(cfg['run_id'], cfg['query_p']), "with query parameter")) except AttributeError: - print "Skipping test GetRunPropertiesById -- missing input parameter" + print("Skipping test GetRunPropertiesById -- missing input parameter") try: self.tests.append((GetProjectById(cfg['project_id'], cfg['query_p']), "with query parameter")) except AttributeError: - print "Skipping test GetProjectById -- missing input parameter" + print("Skipping test GetProjectById -- missing input parameter") try: self.tests.append((GetProjectPropertiesById(cfg['project_id'], cfg['query_p']), "with query parameter")) - except AttributeError: - print "Skipping test GetProjectPropertiesById -- missing input parameter" + except AttributeError: + print("Skipping test GetProjectPropertiesById -- missing input parameter") try: self.tests.append((GetSampleById(cfg['sample_id'], cfg['query_p']), "with query parameter")) except AttributeError: - print "Skipping test GetSampleById -- missing input parameter" + print("Skipping test GetSampleById -- missing input parameter") try: self.tests.append((GetSamplePropertiesById(cfg['sample_id'], cfg['query_p']), "with query parameter")) - except AttributeError: - print "Skipping test GetSamplePropertiesById -- missing input parameter" + except AttributeError: + print("Skipping test GetSamplePropertiesById -- missing input parameter") try: self.tests.append((GetAppResultById(cfg['appresult_id'], cfg['query_p']), "with query parameter")) except AttributeError: - print "Skipping test GetAppResultById -- missing input parameter" + print("Skipping test GetAppResultById -- missing input parameter") try: self.tests.append((GetAppResultPropertiesById(cfg['appresult_id'], cfg['query_p']), "with query parameter")) - except AttributeError: - print "Skipping test GetAppResultPropertiesById -- missing input parameter" + except AttributeError: + print("Skipping test GetAppResultPropertiesById -- missing input parameter") try: self.tests.append((GetFileById(cfg['file_id'], cfg['query_p']), "with query parameter")) except AttributeError: - print "Skipping test GetFileById -- missing input parameter" + print("Skipping test GetFileById -- missing input parameter") try: self.tests.append((GetFilePropertiesById(cfg['file_id'], cfg['query_p']), "with query parameter")) - except AttributeError: - print "Skipping test GetFilePropertiesById -- missing input parameter" + except AttributeError: + print("Skipping test GetFilePropertiesById -- missing input parameter") + - def test_rest_vs_sdk(self): for test in self.tests: - print "\nTesting REST vs SDK for " + test[0].__class__.__name__ + "' with comment '" + test[1] + "'" + print("\nTesting REST vs SDK for " + test[0].__class__.__name__ + "' with comment '" + test[1] + "'") try: test[0].test_rest_vs_sdk() except Exception as e: - print "Exception: " + str(e) + print("Exception: " + str(e)) if __name__ == '__main__': - + # BaseSaceAPI profile (in TestSDK.__init__) must have permission to read the items in cfg cfg = {} cfg['ssn_id'] = '1300310' @@ -393,8 +396,8 @@ def test_rest_vs_sdk(self): 'runs': 'Input.run-id2', #'map': None, 'maps': 'Input.app-result-id2.attributes', - } - cfg['run_id'] = '523524' + } + cfg['run_id'] = '523524' cfg['project_id'] = '2' cfg['sample_id'] = '1021' cfg['appresult_id'] = '21' @@ -404,11 +407,10 @@ def test_rest_vs_sdk(self): cfg['vcf_start'] = '1' cfg['vcf_end'] = '4000' cfg['vcf_format'] = 'txt' # or 'vcf' - - - # Run all tests in the test suite for 'test_app1' + + + # Run all tests in the test suite for 'test_app1' #suite = TestSuite(app_data.test_app1) suite = TestSuite(cfg) suite.add_tests() - suite.test_rest_vs_sdk() - + suite.test_rest_vs_sdk() diff --git a/test/unit_tests.py b/test/unit_tests.py index be807ac..eb2217c 100644 --- a/test/unit_tests.py +++ b/test/unit_tests.py @@ -3,12 +3,12 @@ import sys from tempfile import mkdtemp import shutil -from urlparse import urlparse, urljoin import multiprocessing import hashlib import webbrowser import time import json +import base64 from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI, deviceURL from BaseSpacePy.api.BaseAPI import BaseAPI from BaseSpacePy.api.APIClient import APIClient @@ -19,6 +19,9 @@ +from six.moves.urllib.parse import urlparse, urljoin + + # Dependencies: # ============ # 1. Create a config file in ~/.basespace/unit_tests.cfg that has the credentials for an app on https://portal-hoth.illumina.com; @@ -26,18 +29,18 @@ # 2. Import the following data from Public Dataset 'MiSeq B. cereus demo data' on cloud-hoth.illumina.com: # 2.a. Project name 'BaseSpaceDemo' (Id 596596), and # 2.b. Run name 'BacillusCereus' (Id 555555) -# 3. Download the following fastq file from BaseSpaceDemo's samples section: +# 3. Download the following fastq file from BaseSpaceDemo's samples section: # < https://cloud-hoth.illumina.com/sample/855866/files/tree/BC-12_S12_L001_R2_001.fastq.gz?id=9896135 > # and place it into the 'data' directory of this repository. It's 56MB in size. # # Note that large file upload download tests may take minutes each to complete, and oauth tests will open web browsers. -tconst = { +tconst = { # for download tests 'file_id_small': '9896072', # 2.2 KB, public data B. cereus Project, data/intentisties/basecalls/Alignment/DemultiplexSummaryF1L1.9.txt - 'file_id_large': '9896135', # 55.31 MB public data B. cereus Project, data/intensities/basecalls/BC-12_S12_L001_R2_001.fastq.gz + 'file_id_large': '9896135', # 55.31 MB public data B. cereus Project, data/intensities/basecalls/BC-12_S12_L001_R2_001.fastq.gz 'file_small_md5': '4c3328bcf26ffb54da4de7b3c8879f94', # for file id 9896072 - 'file_large_md5': '9267236a2d870da1d4cb73868bb51b35', # for file id 9896135 + 'file_large_md5': '9267236a2d870da1d4cb73868bb51b35', # for file id 9896135 # for upload tests 'file_small_upload': 'data/test.small.upload.txt', 'file_small_upload_contents': open('data/test.small.upload.txt').read(), @@ -62,7 +65,7 @@ 'PF_count': '446158', 'appresult_id': '1213212', 'appresult_referenced_sample_id': '855855', - #'appsession_id': '1305304', TEMP + #'appsession_id': '1305304', TEMP # for coverage and variant apis 'bam_file_id': '9895890', 'bam_cov_chr_name': 'chr', @@ -71,25 +74,25 @@ 'vcf_file_id': '9895892', 'vcf_chr_name': 'chr', 'vcf_start_coord': '1', - 'vcf_end_coord': '200000', + 'vcf_end_coord': '200000', } class TestFileDownloadMethods(TestCase): ''' Tests methods of File objects ''' - def setUp(self): + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') self.file = self.api.getFileById(tconst['file_id_small']) - self.temp_dir = mkdtemp() - + self.temp_dir = mkdtemp() + def tearDown(self): - shutil.rmtree(self.temp_dir) - + shutil.rmtree(self.temp_dir) + def testDownloadFile(self): new_file = self.file.downloadFile( self.api, - localDir = self.temp_dir, + localDir = self.temp_dir, ) file_path = os.path.join(self.temp_dir, new_file.Name) self.assertTrue(os.path.isfile(file_path)) @@ -98,12 +101,12 @@ def testDownloadFile(self): with open(file_path, "r+b") as fp: self.assertEqual(Utils.md5_for_file(fp), tconst['file_small_md5']) os.remove(file_path) - + def testDownloadFileWithBsDirectoryArg(self): new_file = self.file.downloadFile( self.api, localDir = self.temp_dir, - createBsDir = True, + createBsDir = True, ) file_path = os.path.join(self.temp_dir, new_file.Path) self.assertTrue(os.path.isfile(file_path)) @@ -112,68 +115,68 @@ def testDownloadFileWithBsDirectoryArg(self): with open(file_path, "r+b") as fp: self.assertEqual(Utils.md5_for_file(fp), tconst['file_small_md5']) os.remove(file_path) - + def testDownloadFileWithByteRangeArg(self): new_file = self.file.downloadFile( self.api, localDir = self.temp_dir, - byteRange = [1000,2000] + byteRange = [1000,2000] ) file_path = os.path.join(self.temp_dir, new_file.Name) self.assertTrue(os.path.isfile(file_path)) # confirm file size is correct self.assertEqual(1001, os.stat(file_path).st_size) - os.remove(file_path) + os.remove(file_path) class TestAPIFileUploadMethods_SmallFiles(TestCase): ''' Tests single and multi-part upload methods ''' @classmethod - def setUpClass(cls): + def setUpClass(cls): ''' For all upload unit tests (not per test): Create a new 'unit test' project, or get it if exists, to upload to data to. Then create a new app result in this project, getting a new app session id - ''' - cls.api = BaseSpaceAPI(profile='unit_tests') - cls.proj = cls.api.createProject(tconst['create_project_name']) + ''' + cls.api = BaseSpaceAPI(profile='unit_tests') + cls.proj = cls.api.createProject(tconst['create_project_name']) cls.ar = cls.proj.createAppResult(cls.api, "test upload", "test upload", appSessionId="") - - def test__singlepartFileUpload__(self): + + def test__singlepartFileUpload__(self): testDir = "testSinglePartSmallFileUploadDirectory" fileName = os.path.basename(tconst['file_small_upload']) myFile = self.api.__singlepartFileUpload__( resourceType = 'appresults', resourceId = self.ar.Id, - localPath=tconst['file_small_upload'], - fileName=fileName, - directory=testDir, - contentType=tconst['file_small_upload_content_type']) + localPath=tconst['file_small_upload'], + fileName=fileName, + directory=testDir, + contentType=tconst['file_small_upload_content_type']) self.assertEqual(myFile.Path, os.path.join(testDir, fileName)) self.assertEqual(myFile.Size, tconst['file_small_upload_size']) self.assertEqual(myFile.UploadStatus, 'complete') # test fresh File object newFile = self.api.getFileById(myFile.Id) - self.assertEqual(newFile.Path, os.path.join(testDir, fileName)) + self.assertEqual(newFile.Path, os.path.join(testDir, fileName)) self.assertEqual(newFile.Size, tconst['file_small_upload_size']) - self.assertEqual(newFile.UploadStatus, 'complete') + self.assertEqual(newFile.UploadStatus, 'complete') def testAppResultFileUpload_SmallUpload(self): testDir = "testSmallUploadDirectory" fileName = os.path.basename(tconst['file_small_upload']) myFile = self.api.appResultFileUpload( - Id=self.ar.Id, - localPath=tconst['file_small_upload'], - fileName=fileName, - directory=testDir, - contentType=tconst['file_small_upload_content_type']) + Id=self.ar.Id, + localPath=tconst['file_small_upload'], + fileName=fileName, + directory=testDir, + contentType=tconst['file_small_upload_content_type']) self.assertEqual(myFile.Path, os.path.join(testDir, fileName)) self.assertEqual(myFile.Size, tconst['file_small_upload_size']) self.assertEqual(myFile.UploadStatus, 'complete') # test fresh File object newFile = self.api.getFileById(myFile.Id) - self.assertEqual(newFile.Path, os.path.join(testDir, fileName)) + self.assertEqual(newFile.Path, os.path.join(testDir, fileName)) self.assertEqual(newFile.Size, tconst['file_small_upload_size']) self.assertEqual(newFile.UploadStatus, 'complete') @@ -182,22 +185,22 @@ def test__initiateMultipartFileUpload__(self): file = self.api.__initiateMultipartFileUpload__( resourceType = 'appresults', resourceId = self.ar.Id, - fileName = os.path.basename(tconst['file_small_upload']), + fileName = os.path.basename(tconst['file_small_upload']), directory = testDir, contentType=tconst['file_small_upload_content_type']) - self.assertEqual(file.Name, os.path.basename(tconst['file_small_upload'])) - + self.assertEqual(file.Name, os.path.basename(tconst['file_small_upload'])) + def test__uploadMultipartUnit__(self): testDir = "test__uploadMultipartUnit__" file = self.api.__initiateMultipartFileUpload__( resourceType = 'appresults', resourceId = self.ar.Id, - fileName = os.path.basename(tconst['file_small_upload']), + fileName = os.path.basename(tconst['file_small_upload']), directory = testDir, contentType=tconst['file_small_upload_content_type']) with open(tconst['file_small_upload']) as fp: out = fp.read() - md5 = hashlib.md5(out).digest().encode('base64') + md5 = base64.b64encode(hashlib.md5(out.encode('utf-8')).digest()) response = self.api.__uploadMultipartUnit__( Id = file.Id, partNumber = 1, @@ -205,18 +208,20 @@ def test__uploadMultipartUnit__(self): data = tconst['file_small_upload_contents']) self.assertNotEqual(response, None, 'Upload part failure will return None') self.assertTrue('ETag' in response['Response'], 'Upload part success will contain a Response dict with an ETag element') - + def test__finalizeMultipartFileUpload__(self): testDir = "test__finalizeMultipartFileUpload__" file = self.api.__initiateMultipartFileUpload__( resourceType = 'appresults', resourceId = self.ar.Id, - fileName = os.path.basename(tconst['file_small_upload']), + fileName = os.path.basename(tconst['file_small_upload']), directory = testDir, contentType=tconst['file_small_upload_content_type']) with open(tconst['file_small_upload']) as fp: out = fp.read() - md5 = hashlib.md5(out).digest().encode('base64') + # md5 = hashlib.md5(out).digest().encode('base64') + # import pdb;pdb.set_trace() + md5 = base64.b64encode(hashlib.md5(out.encode('utf-8')).digest()) response = self.api.__uploadMultipartUnit__( Id = file.Id, partNumber = 1, @@ -230,11 +235,11 @@ def testMultiPartFileUpload_SmallPartSizeException(self): myFile = self.api.multipartFileUpload( resourceType = 'appresults', resourceId = self.ar.Id, - localPath=tconst['file_large_upload'], - fileName=os.path.basename(tconst['file_large_upload']), - directory="", - contentType=tconst['file_large_upload_content_type'], - partSize=5, # MB, chunk size + localPath=tconst['file_large_upload'], + fileName=os.path.basename(tconst['file_large_upload']), + directory="", + contentType=tconst['file_large_upload_content_type'], + partSize=5, # MB, chunk size ) def testMultiPartFileUpload_LargePartSizeException(self): @@ -242,21 +247,21 @@ def testMultiPartFileUpload_LargePartSizeException(self): myFile = self.api.multipartFileUpload( resourceType = 'appresults', resourceId = self.ar.Id, - localPath=tconst['file_large_upload'], - fileName=os.path.basename(tconst['file_large_upload']), - directory="", - contentType=tconst['file_large_upload_content_type'], - partSize=26, # MB, chunk size + localPath=tconst['file_large_upload'], + fileName=os.path.basename(tconst['file_large_upload']), + directory="", + contentType=tconst['file_large_upload_content_type'], + partSize=26, # MB, chunk size ) - def testIntegration_SmallFileUploadThenDownload(self): + def testIntegration_SmallFileUploadThenDownload(self): upFile = self.api.appResultFileUpload( - Id=self.ar.Id, - localPath=tconst['file_small_upload'], - fileName=os.path.basename(tconst['file_small_upload']), - directory="test_upload_download_dir", - contentType=tconst['file_small_upload_content_type']) - tempDir = mkdtemp() + Id=self.ar.Id, + localPath=tconst['file_small_upload'], + fileName=os.path.basename(tconst['file_small_upload']), + directory="test_upload_download_dir", + contentType=tconst['file_small_upload_content_type']) + tempDir = mkdtemp() downFile = self.api.fileDownload(upFile.Id, tempDir, createBsDir=True) downPath = os.path.join(tempDir, upFile.Path) self.assertTrue(os.path.isfile(downPath), "Failed to find path %s" % downPath) @@ -264,7 +269,7 @@ def testIntegration_SmallFileUploadThenDownload(self): self.assertEqual(os.path.getsize(tconst['file_small_upload']), os.path.getsize(downPath)) with open(downPath, "r+b") as fp: self.assertEqual(Utils.md5_for_file(fp), tconst['file_small_upload_md5']) - os.remove(downPath) + os.remove(downPath) class TestMultipartFileTransferMethods(TestCase): @@ -280,64 +285,64 @@ class TestAPIFileUploadMethods_LargeFiles(TestCase): Tests multi-part upload methods on large(-ish) files -- may be time consuming ''' @classmethod - def setUpClass(cls): + def setUpClass(cls): ''' For all upload unit tests (not per test): Create a new 'unit test' project, or get it if exists, to upload to data to. Then create a new app result in this project, getting a new app session id - ''' + ''' cls.api = BaseSpaceAPI(profile='unit_tests') cls.proj = cls.api.createProject(tconst['create_project_name']) cls.ar = cls.proj.createAppResult(cls.api, "test upload", "test upload", appSessionId="") - + # @skip('large upload') def testAppResultFileUpload_LargeUpload(self): testDir = "testLargeUploadDirectory" - fileName = os.path.basename(tconst['file_large_upload']) + fileName = os.path.basename(tconst['file_large_upload']) myFile = self.api.appResultFileUpload( - Id=self.ar.Id, - localPath=tconst['file_large_upload'], - fileName=fileName, - directory=testDir, + Id=self.ar.Id, + localPath=tconst['file_large_upload'], + fileName=fileName, + directory=testDir, contentType=tconst['file_small_upload_content_type']) self.assertEqual(myFile.Path, os.path.join(testDir, fileName)) self.assertEqual(myFile.Size, tconst['file_large_upload_size']) self.assertEqual(myFile.UploadStatus, 'complete') # test fresh File object newFile = self.api.getFileById(myFile.Id) - self.assertEqual(newFile.Path, os.path.join(testDir, fileName)) + self.assertEqual(newFile.Path, os.path.join(testDir, fileName)) self.assertEqual(newFile.Size, tconst['file_large_upload_size']) self.assertEqual(newFile.UploadStatus, 'complete') - + # @skip('large upload') def testMultiPartFileUpload(self): testDir = "testMultipartUploadDir" - fileName = os.path.basename(tconst['file_large_upload']) + fileName = os.path.basename(tconst['file_large_upload']) myFile = self.api.multipartFileUpload( resourceType = 'appresults', resourceId = self.ar.Id, - localPath=tconst['file_large_upload'], - fileName=fileName, - directory=testDir, + localPath=tconst['file_large_upload'], + fileName=fileName, + directory=testDir, contentType=tconst['file_large_upload_content_type'], processCount = 4, - partSize= 10, # MB, chunk size + partSize= 10, # MB, chunk size #tempDir = args.temp_dir - ) + ) self.assertEqual(myFile.Size, tconst['file_large_upload_size']) self.assertEqual(myFile.Name, fileName) - self.assertEqual(myFile.Path, os.path.join(testDir, fileName)) - self.assertEqual(myFile.UploadStatus, 'complete') + self.assertEqual(myFile.Path, os.path.join(testDir, fileName)) + self.assertEqual(myFile.UploadStatus, 'complete') # @skip('large upload and download') - def testIntegration_LargeFileUploadThenDownload(self): + def testIntegration_LargeFileUploadThenDownload(self): upFile = self.api.appResultFileUpload( - Id=self.ar.Id, - localPath=tconst['file_large_upload'], - fileName=os.path.basename(tconst['file_large_upload']), - directory="test_upload_download_dir", - contentType=tconst['file_large_upload_content_type']) - tempDir = mkdtemp() + Id=self.ar.Id, + localPath=tconst['file_large_upload'], + fileName=os.path.basename(tconst['file_large_upload']), + directory="test_upload_download_dir", + contentType=tconst['file_large_upload_content_type']) + tempDir = mkdtemp() downFile = self.api.fileDownload(upFile.Id, tempDir, createBsDir=True) downPath = os.path.join(tempDir, upFile.Path) self.assertTrue(os.path.isfile(downPath), "Failed to find path %s" % downPath) @@ -345,26 +350,26 @@ def testIntegration_LargeFileUploadThenDownload(self): self.assertEqual(os.path.getsize(tconst['file_large_upload']), os.path.getsize(downPath)) with open(downPath, "r+b") as fp: self.assertEqual(Utils.md5_for_file(fp), tconst['file_large_upload_md5']) - os.remove(downPath) - + os.remove(downPath) + class TestAPIFileDownloadMethods_SmallFiles(TestCase): ''' Tests single and multi-part download methods ''' - def setUp(self): + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') - self.temp_dir = mkdtemp() - + self.temp_dir = mkdtemp() + def tearDown(self): - shutil.rmtree(self.temp_dir) + shutil.rmtree(self.temp_dir) def test__downloadFile__(self): file_name = 'testfile.abc' bs_file = self.api.getFileById(tconst['file_id_small']) self.api.__downloadFile__( - tconst['file_id_small'], + tconst['file_id_small'], localDir = self.temp_dir, - name = file_name, + name = file_name, ) file_path = os.path.join(self.temp_dir, file_name) self.assertTrue(os.path.isfile(file_path)) @@ -373,43 +378,43 @@ def test__downloadFile__(self): with open(file_path, "r+b") as fp: self.assertEqual(Utils.md5_for_file(fp), tconst['file_small_md5']) os.remove(file_path) - + def test__downloadFile__WithByteRangeArg(self): - file_name = 'testfile.abc' + file_name = 'testfile.abc' self.api.__downloadFile__( - tconst['file_id_large'], + tconst['file_id_large'], localDir = self.temp_dir, name = file_name, - byteRange = [2000,3000] + byteRange = [2000,3000] ) file_path = os.path.join(self.temp_dir, file_name) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isfile(file_path)) self.assertEqual(3001, os.stat(file_path).st_size) # seek() into file, so size is larger os.remove(file_path) def test__downloadFile__WithByteRangeStoredInStandaloneFile(self): file_name = 'testfile.abc' self.api.__downloadFile__( - tconst['file_id_large'], + tconst['file_id_large'], localDir = self.temp_dir, name = file_name, byteRange = [2000,3000], - standaloneRangeFile = True, + standaloneRangeFile = True, ) file_path = os.path.join(self.temp_dir, file_name) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isfile(file_path)) self.assertEqual(1001, os.stat(file_path).st_size) # no seek() into standalone file, so size is only range data os.remove(file_path) - + def test__downloadFile__WithLockArg(self): lock = multiprocessing.Lock() # just testing that passing in a lock won't crash anything file_name = 'testfile.abc' bs_file = self.api.getFileById(tconst['file_id_small']) self.api.__downloadFile__( - tconst['file_id_small'], + tconst['file_id_small'], localDir = self.temp_dir, name = file_name, - lock = lock, + lock = lock, ) file_path = os.path.join(self.temp_dir, file_name) self.assertTrue(os.path.isfile(file_path)) @@ -417,12 +422,12 @@ def test__downloadFile__WithLockArg(self): self.assertEqual(bs_file.Size, os.stat(file_path).st_size) with open(file_path, "r+b") as fp: self.assertEqual(Utils.md5_for_file(fp), tconst['file_small_md5']) - os.remove(file_path) - + os.remove(file_path) + def testFileDownload_SmallFile(self): new_file = self.api.fileDownload( - tconst['file_id_small'], - localDir = self.temp_dir, + tconst['file_id_small'], + localDir = self.temp_dir, ) file_path = os.path.join(self.temp_dir, new_file.Name) self.assertTrue(os.path.isfile(file_path)) @@ -434,9 +439,9 @@ def testFileDownload_SmallFile(self): def testFileDownload_SmallFileWithBsDirectoryArg(self): new_file = self.api.fileDownload( - tconst['file_id_small'], + tconst['file_id_small'], localDir = self.temp_dir, - createBsDir = True, + createBsDir = True, ) file_path = os.path.join(self.temp_dir, new_file.Path) self.assertTrue(os.path.isfile(file_path)) @@ -448,43 +453,43 @@ def testFileDownload_SmallFileWithBsDirectoryArg(self): def testFileDownload_WithByteRangeArg(self): new_file = self.api.fileDownload( - tconst['file_id_large'], + tconst['file_id_large'], localDir = self.temp_dir, - byteRange = [1000,2000] + byteRange = [1000,2000] ) file_path = os.path.join(self.temp_dir, new_file.Name) self.assertTrue(os.path.isfile(file_path)) # confirm file size is correct self.assertEqual(1001, os.stat(file_path).st_size) - os.remove(file_path) + os.remove(file_path) def testFileDownload_LargeByteRangeException(self): with self.assertRaises(ByteRangeException): self.api.fileDownload( - tconst['file_id_large'], + tconst['file_id_large'], localDir = self.temp_dir, - byteRange = [1,10000001] - ) + byteRange = [1,10000001] + ) def testFileDownload_MisorderedByteRangeException(self): with self.assertRaises(ByteRangeException): self.api.fileDownload( - tconst['file_id_large'], + tconst['file_id_large'], localDir = self.temp_dir, - byteRange = [1000, 1] + byteRange = [1000, 1] ) def testFileDownload_PartialByteRangeException(self): with self.assertRaises(ByteRangeException): self.api.fileDownload( - tconst['file_id_large'], + tconst['file_id_large'], localDir = self.temp_dir, - byteRange = [1000] + byteRange = [1000] ) def testMultipartFileDownload_SmallFile(self): new_file = self.api.multipartFileDownload( - tconst['file_id_small'], + tconst['file_id_small'], localDir = self.temp_dir, processCount = 10, partSize = 12 @@ -499,7 +504,7 @@ def testMultipartFileDownload_SmallFile(self): def testMultipartFileDownload_WithBsDirectoryArg(self): new_file = self.api.multipartFileDownload( - tconst['file_id_small'], + tconst['file_id_small'], localDir = self.temp_dir, processCount = 10, partSize = 12, @@ -515,13 +520,13 @@ def testMultipartFileDownload_WithBsDirectoryArg(self): def testMultipartFileDownload_WithTempFileArg(self): new_file = self.api.multipartFileDownload( - tconst['file_id_small'], - localDir = self.temp_dir, + tconst['file_id_small'], + localDir = self.temp_dir, tempDir = self.temp_dir ) file_path = os.path.join(self.temp_dir, new_file.Name) self.assertTrue(os.path.isfile(file_path)) - # confirm file size and md5 are correct + # confirm file size and md5 are correct self.assertEqual(new_file.Size, os.stat(file_path).st_size) fp = open(file_path, "r+b") self.assertEqual(Utils.md5_for_file(fp), tconst['file_small_md5']) @@ -529,14 +534,14 @@ def testMultipartFileDownload_WithTempFileArg(self): def testMultipartFileDownload_WithTempFileAndBsDirArgs(self): new_file = self.api.multipartFileDownload( - tconst['file_id_small'], - localDir = self.temp_dir, + tconst['file_id_small'], + localDir = self.temp_dir, tempDir = self.temp_dir, createBsDir = True, ) file_path = os.path.join(self.temp_dir, new_file.Path) self.assertTrue(os.path.isfile(file_path)) - # confirm file size and md5 are correct + # confirm file size and md5 are correct self.assertEqual(new_file.Size, os.stat(file_path).st_size) fp = open(file_path, "r+b") self.assertEqual(Utils.md5_for_file(fp), tconst['file_small_md5']) @@ -546,18 +551,18 @@ class TestAPIFileDownloadMethods_LargeFiles(TestCase): ''' Tests multi-part download methods on large(-ish) files -- may be time consuming ''' - def setUp(self): + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') - self.temp_dir = mkdtemp() - + self.temp_dir = mkdtemp() + def tearDown(self): - shutil.rmtree(self.temp_dir) + shutil.rmtree(self.temp_dir) # @skip('large download') def testFileDownload_LargeFile(self): new_file = self.api.fileDownload( - tconst['file_id_large'], - localDir = self.temp_dir, + tconst['file_id_large'], + localDir = self.temp_dir, ) file_path = os.path.join(self.temp_dir, new_file.Name) self.assertTrue(os.path.isfile(file_path)) @@ -570,9 +575,9 @@ def testFileDownload_LargeFile(self): # @skip('large download') def testFileDownload_LargeFileWithBsDirectoryArg(self): new_file = self.api.fileDownload( - tconst['file_id_large'], + tconst['file_id_large'], localDir = self.temp_dir, - createBsDir = True, + createBsDir = True, ) file_path = os.path.join(self.temp_dir, new_file.Path) self.assertTrue(os.path.isfile(file_path)) @@ -585,7 +590,7 @@ def testFileDownload_LargeFileWithBsDirectoryArg(self): # @skip('large download') def testMultipartFileDownload_LargeFile(self): new_file = self.api.multipartFileDownload( - tconst['file_id_large'], + tconst['file_id_large'], localDir = self.temp_dir, processCount = 10, partSize = 12 @@ -602,55 +607,55 @@ class TestAppResultMethods(TestCase): ''' Tests AppResult object methods ''' - def setUp(self): + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') self.appResult = self.api.getAppResultById(tconst['appresult_id']) - - def testIsInit(self): + + def testIsInit(self): self.assertEqual(self.appResult.isInit(), True) - + def testIsInitException(self): - appResult = AppResult.AppResult() + appResult = AppResult.AppResult() with self.assertRaises(ModelNotInitializedException): - appResult.isInit() + appResult.isInit() def testGetAccessString(self): self.assertEqual(self.appResult.getAccessStr(), 'write appresult ' + self.appResult.Id) - + def testGetAccessStringWithArg(self): self.assertEqual(self.appResult.getAccessStr('read'), 'read appresult ' + self.appResult.Id) - + def testGetReferencedSamplesIds(self): self.assertEqual(self.appResult.getReferencedSamplesIds(), [tconst['appresult_referenced_sample_id']]) - + def testGetReferencedSamples(self): samples = self.appResult.getReferencedSamples(self.api) self.assertEqual(samples[0].Id, tconst['appresult_referenced_sample_id']) - + def testGetFiles(self): - files = self.appResult.getFiles(self.api) + files = self.appResult.getFiles(self.api) self.assertTrue(hasattr(files[0], 'Id')) def testGetFilesWithQp(self): - files = self.appResult.getFiles(self.api, qp({'Limit':1})) + files = self.appResult.getFiles(self.api, qp({'Limit':1})) self.assertTrue(hasattr(files[0], 'Id')) self.assertEqual(len(files), 1) - + def testUploadFile(self): ''' Create a new 'unit test' project, or get it if exists, to upload to data to. Then create a new appresult in this project, getting a new appsession id Then...upload a file to the new appresult ''' - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test appresult upload", "test appresult upload", appSessionId="") testDir = "testSmallUploadAppResultDirectory" fileName = os.path.basename(tconst['file_small_upload']) myFile = ar.uploadFile( - api=self.api, - localPath=tconst['file_small_upload'], - fileName=fileName, - directory=testDir, + api=self.api, + localPath=tconst['file_small_upload'], + fileName=fileName, + directory=testDir, contentType=tconst['file_small_upload_content_type']) self.assertEqual(myFile.Path, os.path.join(testDir, fileName)) self.assertEqual(myFile.Size, tconst['file_small_upload_size']) @@ -659,72 +664,72 @@ def testUploadFile(self): newFile = self.api.getFileById(myFile.Id) self.assertEqual(newFile.Path, os.path.join(testDir, fileName)) self.assertEqual(newFile.Size, tconst['file_small_upload_size']) - self.assertEqual(newFile.UploadStatus, 'complete') + self.assertEqual(newFile.UploadStatus, 'complete') class TestAPIAppResultMethods(TestCase): ''' Tests API object AppResult methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') def testGetAppResultById(self): appresult = self.api.getAppResultById(tconst['appresult_id']) self.assertTrue(appresult.Id, 'appresult_id') - + def testGetAppResultByIdWithQp(self): appresult = self.api.getAppResultById(tconst['appresult_id'], qp({'Limit':1})) # Limit doesn't make sense here - self.assertTrue(appresult.Id, 'appresult_id') - + self.assertTrue(appresult.Id, 'appresult_id') + def testGetAppResultPropertiesById(self): - props = self.api.getAppResultPropertiesById(tconst['appresult_id']) + props = self.api.getAppResultPropertiesById(tconst['appresult_id']) self.assertTrue(hasattr(props, 'TotalCount')) - + def testGetAppResultPropertiesByIdWithQp(self): props = self.api.getAppResultPropertiesById(tconst['appresult_id'], qp({'Limit':1})) - self.assertTrue(hasattr(props, 'TotalCount')) + self.assertTrue(hasattr(props, 'TotalCount')) self.assertEqual(len(props.Items), 1) def testGetAppResultFilesById(self): - files = self.api.getAppResultFilesById(tconst['appresult_id']) + files = self.api.getAppResultFilesById(tconst['appresult_id']) self.assertTrue(hasattr(files[0], 'Id')) - + def testGetAppResultFilesByIdWithQp(self): - files = self.api.getAppResultFilesById(tconst['appresult_id'], qp({'Limit':1})) + files = self.api.getAppResultFilesById(tconst['appresult_id'], qp({'Limit':1})) self.assertTrue(hasattr(files[0], 'Id')) - self.assertEqual(len(files), 1) - + self.assertEqual(len(files), 1) + def testGetAppResultFiles(self): - files = self.api.getAppResultFiles(tconst['appresult_id']) + files = self.api.getAppResultFiles(tconst['appresult_id']) self.assertTrue(hasattr(files[0], 'Id')) - + def testGetAppResultFilesWithQp(self): - files = self.api.getAppResultFiles(tconst['appresult_id'], qp({'Limit':1})) + files = self.api.getAppResultFiles(tconst['appresult_id'], qp({'Limit':1})) self.assertTrue(hasattr(files[0], 'Id')) - self.assertEqual(len(files), 1) + self.assertEqual(len(files), 1) def testGetAppResultsByProject(self): appresults = self.api.getAppResultsByProject(tconst['project_id']) self.assertTrue(hasattr(appresults[0], 'Id')) - + def testGetAppResultsByProjectWithQp(self): appresults = self.api.getAppResultsByProject(tconst['project_id'], qp({'Limit':1})) self.assertTrue(hasattr(appresults[0], 'Id')) self.assertEqual(len(appresults), 1) - + def testGetAppResultsByProjectWithStatusesArg(self): appresults = self.api.getAppResultsByProject(tconst['project_id'], statuses=['complete']) self.assertTrue(hasattr(appresults[0], 'Id')) - + def testCreateAppResultNewAppSsn(self): ''' Create a new 'unit test' project, or get it if exists. - Create a new app result that creates a new app ssn. + Create a new app result that creates a new app ssn. ''' - proj = self.api.createProject(tconst['create_project_name']) - ar = self.api.createAppResult(proj.Id, name="test create appresult new ssn", + proj = self.api.createProject(tconst['create_project_name']) + ar = self.api.createAppResult(proj.Id, name="test create appresult new ssn", desc="test create appresult new ssn", appSessionId="") - self.assertTrue(hasattr(ar, 'Id')) + self.assertTrue(hasattr(ar, 'Id')) def testCreateAppResultCredentialsAppSsn(self): ''' @@ -733,69 +738,69 @@ def testCreateAppResultCredentialsAppSsn(self): then create a new api obj with the new ssn, then create an appresult in the new ssn ''' - proj = self.api.createProject(tconst['create_project_name']) - ar = self.api.createAppResult(proj.Id, name="test create appresult creds ssn", + proj = self.api.createProject(tconst['create_project_name']) + ar = self.api.createAppResult(proj.Id, name="test create appresult creds ssn", desc="test create appresult creds ssn", appSessionId="") #url = urlparse(self.api.apiClient.apiServer) #newApiServer = url.scheme + "://" + url.netloc - #new_api = BaseSpaceAPI(self.api.key, self.api.secret, newApiServer, + #new_api = BaseSpaceAPI(self.api.key, self.api.secret, newApiServer, new_api = BaseSpaceAPI(self.api.key, self.api.secret, self.api.apiServer, self.api.version, ar.AppSession.Id, self.api.getAccessToken()) - ar2 = new_api.createAppResult(proj.Id, name="test create appresult creds ssn 2", + ar2 = new_api.createAppResult(proj.Id, name="test create appresult creds ssn 2", desc="test create appresult creds ssn 2") self.assertTrue(hasattr(ar2, 'Id')) - + def testCreateAppResultProvidedAppSsn(self): ''' Create a new app result that creates a new app ssn, then create a new api obj with the new ssn, then create an appresult in the new ssn ''' - proj = self.api.createProject(tconst['create_project_name']) - ar = self.api.createAppResult(proj.Id, name="test create appresult provided ssn", + proj = self.api.createProject(tconst['create_project_name']) + ar = self.api.createAppResult(proj.Id, name="test create appresult provided ssn", desc="test create appresult provided ssn", appSessionId="") - ar2 = self.api.createAppResult(proj.Id, name="test create appresult provided ssn 2", + ar2 = self.api.createAppResult(proj.Id, name="test create appresult provided ssn 2", desc="test create appresult provided ssn 2", appSessionId=ar.AppSession.Id) self.assertTrue(hasattr(ar2, 'Id')) - - # Note that appResultFileUpload() is tested with other file upload methods + + # Note that appResultFileUpload() is tested with other file upload methods # (in a separate suite: TestAPIUploadMethods) - + class TestRunMethods(TestCase): ''' Tests Run object methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') - self.run = self.api.getRunById(tconst['run_id']) + self.run = self.api.getRunById(tconst['run_id']) - def testIsInit(self): + def testIsInit(self): self.assertEqual(self.run.isInit(), True) - + def testIsInitException(self): run = Run.Run() with self.assertRaises(ModelNotInitializedException): - run.isInit() + run.isInit() def testGetAccessString(self): self.assertEqual(self.run.getAccessStr(), 'write run ' + self.run.Id) - + def testGetAccessStringWithArg(self): self.assertEqual(self.run.getAccessStr('read'), 'read run ' + self.run.Id) def testRunGetFiles(self): - rf = self.run.getFiles(self.api) + rf = self.run.getFiles(self.api) self.assertTrue(hasattr(rf[0], 'Id')) - + def testRunGetFilesWithQp(self): - rf = self.run.getFiles(self.api, qp({'Limit':200})) + rf = self.run.getFiles(self.api, qp({'Limit':200})) self.assertTrue(hasattr(rf[0], 'Id')) self.assertEqual(len(rf), 200) def testRunSamples(self): - rs = self.run.getSamples(self.api) + rs = self.run.getSamples(self.api) self.assertTrue(hasattr(rs[0], 'Id')) - + def testRunSamplesWithQp(self): rs = self.run.getSamples(self.api, qp({'Limit':1})) self.assertTrue(hasattr(rs[0], 'Id')) @@ -804,8 +809,8 @@ def testRunSamplesWithQp(self): class TestAPIRunMethods(TestCase): ''' Tests API object Run methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') def testGetAccessibleRunsByUser(self): @@ -816,37 +821,37 @@ def testGetAccessibleRunsByUserWithQp(self): runs = self.api.getAccessibleRunsByUser(qp({'Limit':500})) run = next(r for r in runs if r.Id == tconst['run_id']) self.assertTrue(run.Id, tconst['run_id']) - - def testGetRunById(self): - rf = self.api.getRunById(tconst['run_id']) + + def testGetRunById(self): + rf = self.api.getRunById(tconst['run_id']) self.assertEqual(rf.Id, tconst['run_id']) - - def testGetRunByIdWithQp(self): - rf = self.api.getRunById(tconst['run_id'], qp({'Limit':1})) # limit doesn't make much sense here + + def testGetRunByIdWithQp(self): + rf = self.api.getRunById(tconst['run_id'], qp({'Limit':1})) # limit doesn't make much sense here self.assertEqual(rf.Id, tconst['run_id']) - - def testGetRunPropertiesById(self): - props = self.api.getRunPropertiesById(tconst['run_id']) - self.assertTrue(hasattr(props, 'TotalCount')) - - def testGetRunPropertiesByIdWithQp(self): - props = self.api.getRunPropertiesById(tconst['run_id'], qp({'Limit':1})) + + def testGetRunPropertiesById(self): + props = self.api.getRunPropertiesById(tconst['run_id']) + self.assertTrue(hasattr(props, 'TotalCount')) + + def testGetRunPropertiesByIdWithQp(self): + props = self.api.getRunPropertiesById(tconst['run_id'], qp({'Limit':1})) self.assertTrue(hasattr(props, 'TotalCount')) self.assertEqual(len(props.Items), 1) - - def testGetRunFilesById(self): - rf = self.api.getRunFilesById(tconst['run_id']) + + def testGetRunFilesById(self): + rf = self.api.getRunFilesById(tconst['run_id']) self.assertTrue(hasattr(rf[0], 'Id')) - + def testGetRunFilesByIdWithQp(self): rf = self.api.getRunFilesById(tconst['run_id'], qp({'Limit':1})) self.assertTrue(hasattr(rf[0], 'Id')) - self.assertEqual(len(rf), 1) + self.assertEqual(len(rf), 1) def testRunSamplesById(self): - rs = self.api.getRunSamplesById(tconst['run_id']) + rs = self.api.getRunSamplesById(tconst['run_id']) self.assertTrue(hasattr(rs[0], 'Id')) - + def testRunSamplesByIdWithQp(self): rs = self.api.getRunSamplesById(tconst['run_id'], qp({'Limit':1})) self.assertTrue(hasattr(rs[0], 'Id')) @@ -855,35 +860,35 @@ def testRunSamplesByIdWithQp(self): class TestSampleMethods(TestCase): ''' Tests Sample object methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') self.sample = self.api.getSampleById(tconst['sample_id']) - - def testIsInit(self): + + def testIsInit(self): self.assertEqual(self.sample.isInit(), True) - + def testIsInitException(self): sample = Sample.Sample() with self.assertRaises(ModelNotInitializedException): - sample.isInit() + sample.isInit() def testGetAccessString(self): self.assertEqual(self.sample.getAccessStr(), 'write sample ' + self.sample.Id) - + def testGetAccessStringWithArg(self): self.assertEqual(self.sample.getAccessStr('read'), 'read sample ' + self.sample.Id) - + def testGetReferencedAppResults(self): ars = self.sample.getReferencedAppResults(self.api) self.assertTrue(hasattr(ars[0], 'Id'), "Referenced AppResult should have an Id (assuming this Sample has been analyzed)") - + def testGetFiles(self): - files = self.sample.getFiles(self.api) + files = self.sample.getFiles(self.api) self.assertTrue(hasattr(files[0], "Id")) def testGetFilesWithQp(self): - files = self.sample.getFiles(self.api, qp({'Limit':1})) + files = self.sample.getFiles(self.api, qp({'Limit':1})) self.assertTrue(hasattr(files[0], "Id")) self.assertEqual(len(files), 1) @@ -900,10 +905,10 @@ def testUploadFile(self): testDir = "testLargeUploadSampleDirectory" fileName = os.path.basename(tconst['file_large_upload']) myFile = s.uploadFile( - api=self.api, - localPath=tconst['file_large_upload'], - fileName=fileName, - directory=testDir, + api=self.api, + localPath=tconst['file_large_upload'], + fileName=fileName, + directory=testDir, contentType=tconst['file_large_upload_content_type']) self.assertEqual(myFile.Path, os.path.join(testDir, fileName)) self.assertEqual(myFile.Size, tconst['file_large_upload_size']) @@ -920,7 +925,7 @@ class TestAPISampleMethods(TestCase): ''' def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') - + def testGetSamplesByProject(self): samples = self.api.getSamplesByProject(tconst['project_id']) self.assertIsInstance(int(samples[0].Id), int) @@ -928,29 +933,29 @@ def testGetSamplesByProject(self): def testGetSamplesByProjectWithQp(self): samples = self.api.getSamplesByProject(tconst['project_id'], qp({'Limit':1})) self.assertIsInstance(int(samples[0].Id), int) - self.assertEqual(len(samples), 1) + self.assertEqual(len(samples), 1) - def testGetSampleById(self): + def testGetSampleById(self): sample = self.api.getSampleById(tconst['sample_id']) self.assertEqual(sample.Id, tconst['sample_id']) - def testGetSampleByIdWithQp(self): + def testGetSampleByIdWithQp(self): sample = self.api.getSampleById(tconst['sample_id'], qp({'Limit':1})) # Limit doesn't make much sense here - self.assertEqual(sample.Id, tconst['sample_id']) - + self.assertEqual(sample.Id, tconst['sample_id']) + def testGetSamplePropertiesById(self): props = self.api.getSamplePropertiesById(tconst['sample_id']) - self.assertTrue(hasattr(props, 'TotalCount')) + self.assertTrue(hasattr(props, 'TotalCount')) def testGetSamplePropertiesByIdWithQp(self): props = self.api.getSamplePropertiesById(tconst['sample_id'], qp({'Limit':1})) - self.assertTrue(hasattr(props, 'TotalCount')) + self.assertTrue(hasattr(props, 'TotalCount')) self.assertEqual(len(props.Items), 1) - + def testGetSampleFilesById(self): files = self.api.getSampleFilesById(tconst['sample_id']) self.assertTrue(hasattr(files[0], 'Id')) - + def testGetSampleFilesByIdWithQp(self): files = self.api.getSampleFilesById(tconst['sample_id'], qp({'Limit':1})) self.assertTrue(hasattr(files[0], 'Id')) @@ -963,10 +968,10 @@ class TestProjectMethods(TestCase): def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') self.project = self.api.getProjectById(tconst['project_id']) - - def testIsInit(self): + + def testIsInit(self): self.assertEqual(self.project.isInit(), True) - + def testIsInitException(self): project = Project.Project() with self.assertRaises(ModelNotInitializedException): @@ -974,14 +979,14 @@ def testIsInitException(self): def testGetAccessString(self): self.assertEqual(self.project.getAccessStr(), 'write project ' + self.project.Id) - + def testGetAccessStringWithArg(self): - self.assertEqual(self.project.getAccessStr('read'), 'read project ' + self.project.Id) - + self.assertEqual(self.project.getAccessStr('read'), 'read project ' + self.project.Id) + def testGetAppResults(self): appresults = self.project.getAppResults(self.api) self.assertTrue(hasattr(appresults[0], 'Id')) - + def testGetAppResultsWithOptionalArgs(self): appresults = self.project.getAppResults(self.api, qp({'Limit':1}), statuses=['complete']) self.assertTrue(hasattr(appresults[0], 'Id')) @@ -990,7 +995,7 @@ def testGetAppResultsWithOptionalArgs(self): def testGetSamples(self): samples = self.project.getSamples(self.api) self.assertIsInstance(int(samples[0].Id), int) - + def testGetSamplesWithOptionalArgs(self): samples = self.project.getSamples(self.api, qp({'Limit':1})) self.assertIsInstance(int(samples[0].Id), int) @@ -1003,27 +1008,27 @@ def testCreateAppResult(self): then create a new api obj with the new ssn, then create an appresult in the new ssn ''' - proj = self.api.createProject(tconst['create_project_name']) - ar = proj.createAppResult(self.api, name="test create appresult creds ssn, project obj", + proj = self.api.createProject(tconst['create_project_name']) + ar = proj.createAppResult(self.api, name="test create appresult creds ssn, project obj", desc="test create appresult creds ssn, project obj", appSessionId="") #url = urlparse(self.api.apiClient.apiServer) - #newApiServer = url.scheme + "://" + url.netloc - #new_api = BaseSpaceAPI(self.api.key, self.api.secret, newApiServer, - new_api = BaseSpaceAPI(self.api.key, self.api.secret, self.api.apiServer, + #newApiServer = url.scheme + "://" + url.netloc + #new_api = BaseSpaceAPI(self.api.key, self.api.secret, newApiServer, + new_api = BaseSpaceAPI(self.api.key, self.api.secret, self.api.apiServer, self.api.version, ar.AppSession.Id, self.api.getAccessToken()) - ar2 = proj.createAppResult(new_api, name="test create appresult creds ssn, project obj 2", + ar2 = proj.createAppResult(new_api, name="test create appresult creds ssn, project obj 2", desc="test create appresult creds ssn, proejct obj 2") - self.assertTrue(hasattr(ar2, 'Id')) + self.assertTrue(hasattr(ar2, 'Id')) def testCreateAppResultWithOptionalArgs(self): ''' Create a new 'unit test' project, or get it if exists. - Create a new app result that creates a new app ssn. + Create a new app result that creates a new app ssn. ''' - proj = self.api.createProject(tconst['create_project_name']) - ar = proj.createAppResult(self.api, name="test create appresult new ssn, project obj", + proj = self.api.createProject(tconst['create_project_name']) + ar = proj.createAppResult(self.api, name="test create appresult new ssn, project obj", desc="test create appresult new ssn, project obj", samples=[], appSessionId="") - self.assertTrue(hasattr(ar, 'Id')) + self.assertTrue(hasattr(ar, 'Id')) def testCreateSample(self): ''' @@ -1032,7 +1037,7 @@ def testCreateSample(self): then create a new api obj with the new ssn, then create a sample in the new ssn ''' - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) s = proj.createSample(self.api, "SRA123456", "SRA Import", 1, tconst['create_sample_name'], [tconst['read_length']], tconst['raw_count'], tconst['PF_count'], appSessionId="") @@ -1046,80 +1051,80 @@ def testCreateSample(self): class TestAPIProjectMethods(TestCase): ''' Tests API Project object methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') def testCreateProject(self): proj = self.api.createProject(tconst['create_project_name']) - self.assertEqual(proj.Name, tconst['create_project_name']) - + self.assertEqual(proj.Name, tconst['create_project_name']) + def testGetProjectById(self): proj = self.api.getProjectById(tconst['project_id']) self.assertEqual(proj.Id, tconst['project_id']) def testGetProjectByIdWithQp(self): proj = self.api.getProjectById(tconst['project_id'], qp({'Limit':1})) # Limit doesn't make sense here - self.assertEqual(proj.Id, tconst['project_id']) + self.assertEqual(proj.Id, tconst['project_id']) def testGetProjectPropertiesById(self): props = self.api.getProjectPropertiesById(tconst['project_id']) - self.assertTrue(hasattr(props, 'TotalCount')) + self.assertTrue(hasattr(props, 'TotalCount')) def testGetProjectPropertiesByIdWithQp(self): - props = self.api.getProjectPropertiesById(tconst['project_id'], qp({'Limit':1})) - self.assertTrue(hasattr(props, 'TotalCount')) + props = self.api.getProjectPropertiesById(tconst['project_id'], qp({'Limit':1})) + self.assertTrue(hasattr(props, 'TotalCount')) # test project has no properties, so can't test Limit def testGetProjectByUser(self): - projects = self.api.getProjectByUser() + projects = self.api.getProjectByUser() self.assertTrue(hasattr(projects[0], 'Id')) - + def testGetProjectByUserWithQp(self): - projects = self.api.getProjectByUser(qp({'Limit':1})) - self.assertTrue(hasattr(projects[0], 'Id')) + projects = self.api.getProjectByUser(qp({'Limit':1})) + self.assertTrue(hasattr(projects[0], 'Id')) class TestUserMethods(TestCase): ''' Tests User object methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') self.user = self.api.getUserById('current') - - def testIsInit(self): + + def testIsInit(self): self.assertEqual(self.user.isInit(), True) - + def testIsInitException(self): user = User.User() with self.assertRaises(ModelNotInitializedException): user.isInit() - + def testGetProjects(self): - projects = self.user.getProjects(self.api) + projects = self.user.getProjects(self.api) self.assertTrue(hasattr(projects[0], 'Id')) - + def testGetProjectsWithQp(self): - projects = self.user.getProjects(self.api, queryPars=qp({'Limit':1})) + projects = self.user.getProjects(self.api, queryPars=qp({'Limit':1})) self.assertTrue(hasattr(projects[0], 'Id')) self.assertTrue(len(projects), 1) - + def testGetRuns(self): - runs = self.user.getRuns(self.api) + runs = self.user.getRuns(self.api) self.assertTrue(hasattr(runs[0], 'Id')) - + def testGetRunsWithQp(self): - runs = self.user.getRuns(self.api, queryPars=qp({'Limit':1})) + runs = self.user.getRuns(self.api, queryPars=qp({'Limit':1})) self.assertTrue(hasattr(runs[0], 'Id')) self.assertTrue(len(runs), 1) class TestAPIUserMethods(TestCase): ''' Tests API User object methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') - + def testGetUserById(self): user = self.api.getUserById('current') self.assertTrue(hasattr(user, 'Id'), 'User object should contain Id attribute') @@ -1127,32 +1132,32 @@ def testGetUserById(self): class TestFileMethods(TestCase): ''' Tests File object methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') self.file = self.api.getFileById(tconst['file_id_small']) - - def testIsInit(self): + + def testIsInit(self): self.assertEqual(self.file.isInit(), True) - + def testIsInitException(self): file = File.File() with self.assertRaises(ModelNotInitializedException): file.isInit() - - # not testing isValidFileOption() -- deprecated + + # not testing isValidFileOption() -- deprecated # downloadFile() is tested in a separate suite - + def testGetFileUrl(self): url = self.file.getFileUrl(self.api) url_parts = urlparse(url) self.assertEqual(url_parts.scheme, 'https') - + def testGetFileS3metadata(self): - meta = self.file.getFileS3metadata(self.api) + meta = self.file.getFileS3metadata(self.api) self.assertTrue('url' in meta) - self.assertTrue('etag' in meta) + self.assertTrue('etag' in meta) def testGetIntervalCoverage(self): bam = self.api.getFileById(tconst['bam_file_id']) @@ -1168,21 +1173,21 @@ def testGetCoverageMeta(self): cov_meta = bam.getCoverageMeta( self.api, Chrom = tconst['bam_cov_chr_name'] ) - self.assertTrue(hasattr(cov_meta, 'MaxCoverage')) - + self.assertTrue(hasattr(cov_meta, 'MaxCoverage')) + def testFilterVariant(self): vcf = self.api.getFileById(tconst['vcf_file_id']) vars = vcf.filterVariant( - self.api, + self.api, Chrom = tconst['vcf_chr_name'], StartPos = tconst['vcf_start_coord'], - EndPos = tconst['vcf_end_coord'], ) + EndPos = tconst['vcf_end_coord'], ) self.assertEqual(vars[0].CHROM, tconst['vcf_chr_name']) - + def testFilterVariantWithQp(self): vcf = self.api.getFileById(tconst['vcf_file_id']) vars = vcf.filterVariant( - self.api, + self.api, Chrom = tconst['vcf_chr_name'], StartPos = tconst['vcf_start_coord'], EndPos = tconst['vcf_end_coord'], @@ -1190,18 +1195,18 @@ def testFilterVariantWithQp(self): queryPars = qp({'Limit':1}) ) self.assertEqual(vars[0].CHROM, tconst['vcf_chr_name']) self.assertEqual(len(vars), 1) - + def testFilterVariantReturnVCFString(self): vcf = self.api.getFileById(tconst['vcf_file_id']) with self.assertRaises(NotImplementedError): # for now... vars = vcf.filterVariant( - self.api, + self.api, Chrom = tconst['vcf_chr_name'], StartPos = tconst['vcf_start_coord'], EndPos = tconst['vcf_end_coord'], Format = 'vcf') - #self.assertEqual(type(vars), str) - + #self.assertEqual(type(vars), str) + def testGetVariantMeta(self): vcf = self.api.getFileById(tconst['vcf_file_id']) hdr = vcf.getVariantMeta(self.api) @@ -1216,22 +1221,22 @@ def testGetVariantMetaReturnVCFString(self): class TestAPIFileMethods(TestCase): ''' Tests API File object methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') - + def testGetFileById(self): file = self.api.getFileById(tconst['file_id_small']) self.assertTrue(file.Id, tconst['file_id_small']) def testGetFileByIdWithQp(self): file = self.api.getFileById(tconst['file_id_small'], qp({'Limit':1})) # Limit doesn't make much sense here - self.assertEqual(file.Id, tconst['file_id_small']) + self.assertEqual(file.Id, tconst['file_id_small']) def testGetFilesBySample(self): files = self.api.getFilesBySample(tconst['sample_id']) self.assertTrue(hasattr(files[0], 'Id')) - + def testGetFilesBySampleWithQp(self): files = self.api.getFilesBySample(tconst['sample_id'], qp({'Limit':1})) self.assertTrue(hasattr(files[0], 'Id')) @@ -1240,7 +1245,7 @@ def testGetFilesBySampleWithQp(self): def testGetFilePropertiesById(self): props = self.api.getFilePropertiesById(tconst['file_id_small']) self.assertTrue(hasattr(props, 'TotalCount')) - + def testGetFilePropertiesByIdWithQp(self): props = self.api.getFilePropertiesById(tconst['file_id_small'], qp({'Limit':1})) self.assertTrue(hasattr(props, 'TotalCount')) @@ -1250,45 +1255,45 @@ def testFileUrl(self): url = self.api.fileUrl(tconst['file_id_small']) url_parts = urlparse(url) self.assertEqual(url_parts.scheme, 'https') - + def testFileS3metadata(self): - meta = self.api.fileS3metadata(tconst['file_id_small']) + meta = self.api.fileS3metadata(tconst['file_id_small']) self.assertTrue('url' in meta) self.assertTrue('etag' in meta) - # api file upload/download methods are tested in a separate suite: - # __initiateMultipartFileUpload__() - # __uploadMultipartUnit__() - # __finalizeMultipartFileUpload__() - # __singlepartFileUpload__() - # multipartFileUpload() - + # api file upload/download methods are tested in a separate suite: + # __initiateMultipartFileUpload__() + # __uploadMultipartUnit__() + # __finalizeMultipartFileUpload__() + # __singlepartFileUpload__() + # multipartFileUpload() + # __downloadFile__() # fileDownload() - # multipartFileDownload() + # multipartFileDownload() class TestAppSessionSemiCompactMethods(TestCase): ''' Tests AppSessionSemiCompact object methods - ''' + ''' @classmethod - def setUpClass(cls): + def setUpClass(cls): cls.api = BaseSpaceAPI(profile='unit_tests') # create an app session, since the client key and secret must match those of the ssn application - cls.proj = cls.api.createProject(tconst['create_project_name']) + cls.proj = cls.api.createProject(tconst['create_project_name']) cls.ar = cls.proj.createAppResult(cls.api, "test AppSessionSemiCompact Methods", "test AppSessionSemiCompact Methods", appSessionId="") cls.ssn = cls.ar.AppSession # this is an AppSessionSemiCompact instance - def testIsInit(self): + def testIsInit(self): self.assertEqual(self.ssn.isInit(), True) - + def testIsInitException(self): - ssn = AppSessionSemiCompact.AppSessionSemiCompact() + ssn = AppSessionSemiCompact.AppSessionSemiCompact() with self.assertRaises(ModelNotInitializedException): - ssn.isInit() - + ssn.isInit() + def testCanWorkOn(self): - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test canWorkOn()", "test canWorkOn()", appSessionId="") self.assertEqual(ar.AppSession.canWorkOn(), True) ar.AppSession.setStatus(self.api, 'NeedsAttention', "Will you look into this?") @@ -1296,28 +1301,28 @@ def testCanWorkOn(self): ar.AppSession.setStatus(self.api, 'TimedOut', "This is taking forever") self.assertEqual(ar.AppSession.canWorkOn(), True) ar.AppSession.setStatus(self.api, 'Complete', "Time to wrap things up") - self.assertEqual(ar.AppSession.canWorkOn(), False) + self.assertEqual(ar.AppSession.canWorkOn(), False) def testCanWorkOn_Aborted(self): - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test canWorkOn() Aborted", "test canWorkOn() Aborted", appSessionId="") self.assertEqual(ar.AppSession.canWorkOn(), True) ar.AppSession.setStatus(self.api, 'Aborted', "Abandon Ship!") - self.assertEqual(ar.AppSession.canWorkOn(), False) - + self.assertEqual(ar.AppSession.canWorkOn(), False) + def setStatus(self): status = 'Complete' statusSummary = "Let's go home now" - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test setStatus()", "test setStatus()", appSessionId="") ar.AppSession.setStatus(self.api, status, statusSummary) self.assertEqual(ar.AppSession.Status, status) self.assertEqual(ar.AppSession.StatusSummary, statusSummary) - + def testSetStatus_CompleteStatusException(self): status = 'Complete' statusSummary = "Let's go" - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test setStatus() Complete exception", "test setStatus() Complete exception", appSessionId="") ar.AppSession.setStatus(self.api, status, statusSummary) status = 'Aborted' @@ -1328,36 +1333,36 @@ def testSetStatus_CompleteStatusException(self): def testSetStatus_AbortedStatusException(self): status = 'Aborted' statusSummary = "Let's go" - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test setStatus() aborted exception", "test setStatus() aborted exception", appSessionId="") ar.AppSession.setStatus(self.api, status, statusSummary) status = 'Running' statusSummary = "I thought everything was peachy?" with self.assertRaises(AppSessionException): ar.AppSession.setStatus(self.api, status, statusSummary) - + class TestAppSessionMethods(TestCase): ''' Tests AppSession object methods - ''' + ''' @classmethod - def setUpClass(cls): + def setUpClass(cls): cls.api = BaseSpaceAPI(profile='unit_tests') # create an app session, since the client key and secret must match those of the ssn application - cls.proj = cls.api.createProject(tconst['create_project_name']) + cls.proj = cls.api.createProject(tconst['create_project_name']) cls.ar = cls.proj.createAppResult(cls.api, "test AppSession Methods", "test AppSession Methods", appSessionId="") - cls.ssn = cls.api.getAppSessionById(cls.ar.AppSession.Id) # this is an AppSession instance + cls.ssn = cls.api.getAppSessionById(cls.ar.AppSession.Id) # this is an AppSession instance - def testIsInit(self): + def testIsInit(self): self.assertEqual(self.ssn.isInit(), True) - + def testIsInitException(self): - ssn = AppSession.AppSession() + ssn = AppSession.AppSession() with self.assertRaises(ModelNotInitializedException): - ssn.isInit() - + ssn.isInit() + def testCanWorkOn(self): - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test canWorkOn()", "test canWorkOn()", appSessionId="") ssn = self.api.getAppSessionById(ar.AppSession.Id) self.assertEqual(ssn.canWorkOn(), True) @@ -1366,32 +1371,32 @@ def testCanWorkOn(self): ssn.setStatus(self.api, 'TimedOut', "This is taking forever") self.assertEqual(ssn.canWorkOn(), True) ssn.setStatus(self.api, 'Complete', "Time to wrap things up") - self.assertEqual(ssn.canWorkOn(), False) + self.assertEqual(ssn.canWorkOn(), False) def testCanWorkOn_Aborted(self): - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test canWorkOn() Aborted", "test canWorkOn() Aborted", appSessionId="") ssn = self.api.getAppSessionById(ar.AppSession.Id) self.assertEqual(ssn.canWorkOn(), True) ssn.setStatus(self.api, 'Aborted', "Abandon Ship!") - self.assertEqual(ssn.canWorkOn(), False) - + self.assertEqual(ssn.canWorkOn(), False) + def setStatus(self): status = 'Complete' statusSummary = "Let's go home now" - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test setStatus()", "test setStatus()", appSessionId="") ssn = self.api.getAppSessionById(ar.AppSession.Id) ssn.setStatus(self.api, status, statusSummary) self.assertEqual(ssn.Status, status) self.assertEqual(ssn.StatusSummary, statusSummary) - + def testSetStatus_CompleteStatusException(self): status = 'Complete' statusSummary = "Let's go" - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test setStatus() Complete exception", "test setStatus() Complete exception", appSessionId="") - ssn = self.api.getAppSessionById(ar.AppSession.Id) + ssn = self.api.getAppSessionById(ar.AppSession.Id) ssn.setStatus(self.api, status, statusSummary) status = 'Aborted' statusSummary = '(Too) late breaking changes' @@ -1401,19 +1406,19 @@ def testSetStatus_CompleteStatusException(self): def testSetStatus_AbortedStatusException(self): status = 'Aborted' statusSummary = "Let's go" - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test setStatus() aborted exception", "test setStatus() aborted exception", appSessionId="") - ssn = self.api.getAppSessionById(ar.AppSession.Id) + ssn = self.api.getAppSessionById(ar.AppSession.Id) ssn.setStatus(self.api, status, statusSummary) status = 'Running' statusSummary = "I thought everything was peachy?" with self.assertRaises(AppSessionException): ssn.setStatus(self.api, status, statusSummary) - - def test__deserializeReferences__(self): + + def test__deserializeReferences__(self): asla = AppSessionLaunchObject.AppSessionLaunchObject() asla.Type = 'Project' - asla.Content = { "Id": "123", + asla.Content = { "Id": "123", "UserOwnedBy": {"Id": "321", "Href": "v1pre3/users/321", "Name": "Jay Flatley" }, @@ -1429,44 +1434,44 @@ class TestAppSessionLaunchObjectMethods(TestCase): ''' Tests AppSessionLaunchObject object methods ''' - def setUp(self): - self.api = BaseSpaceAPI(profile='unit_tests') - + def setUp(self): + self.api = BaseSpaceAPI(profile='unit_tests') + def test__deserializeObject__(self): asla = AppSessionLaunchObject.AppSessionLaunchObject() asla.Type = 'Project' - asla.Content = { "Id": "123", + asla.Content = { "Id": "123", "UserOwnedBy": {"Id": "321", "Href": "v1pre3/users/321", "Name": "Jay Flatley" }, "Href": "v1pre3/projects/123", "Name": "Project Boomtown", - "DataCreated": "2020-01-01T01:01:01.0000000" } + "DataCreated": "2020-01-01T01:01:01.0000000" } asla.__deserializeObject__(self.api) self.assertEqual(asla.Content.Id, "123") - + class TestAPIAppSessionMethods(TestCase): ''' Tests API AppSession object methods - ''' + ''' @classmethod - def setUpClass(cls): + def setUpClass(cls): cls.api = BaseSpaceAPI(profile='unit_tests') # create an app session, since the client key and secret must match those of the ssn application - cls.proj = cls.api.createProject(tconst['create_project_name']) + cls.proj = cls.api.createProject(tconst['create_project_name']) cls.ar = cls.proj.createAppResult(cls.api, "test API AppSession Methods", "test API AppSession Methods", appSessionId="") cls.ssn = cls.ar.AppSession - def test__deserializeAppSessionResponse__(self): + def test__deserializeAppSessionResponse__(self): # very similar to 2nd half of BaseAPI.__singleRequest__() references = [ { "Type": "Project", - "Href": "v1pre3/projects/321", - "Content": {"Id": "321", } } ] + "Href": "v1pre3/projects/321", + "Content": {"Id": "321", } } ] ssn_dict = { "ResponseStatus": {}, - "Notifications": {}, + "Notifications": {}, "Response": {"Id": "123", "Href": "v1pre3/appsessions/123", - "References": references, } } + "References": references, } } ssn = self.api.__deserializeAppSessionResponse__(ssn_dict) self.assertEqual(ssn.Id, "123") self.assertEqual(ssn.References[0].Content.Id, "321") @@ -1474,9 +1479,9 @@ def test__deserializeAppSessionResponse__(self): def test__deserializeAppSessionResponse__ErrorCodeException(self): ssn_dict = { "ResponseStatus": { "ErrorCode": "666", "Message": "We are dying" } } with self.assertRaises(AppSessionException): - self.api.__deserializeAppSessionResponse__(ssn_dict) - - def testGetAppSessionById(self): + self.api.__deserializeAppSessionResponse__(ssn_dict) + + def testGetAppSessionById(self): ssn = self.api.getAppSessionById(self.ssn.Id) self.assertEqual(ssn.Id, self.ssn.Id) @@ -1491,7 +1496,7 @@ def testGetAppSessionWithId(self): def testGetAppSessionPropertiesById(self): props = self.api.getAppSessionPropertiesById(self.ssn.Id) - self.assertTrue(any((prop.Items[0].Id == self.ar.Id) for prop in props.Items if prop.Name == "Output.AppResults")) + self.assertTrue(any((prop.Items[0].Id == self.ar.Id) for prop in props.Items if prop.Name == "Output.AppResults")) def testGetAppSessionPropertiesByIdWithQp(self): props = self.api.getAppSessionPropertiesById(self.ssn.Id, qp({'Limit':1})) @@ -1515,7 +1520,7 @@ def testGetAppSessionInputsById(self): # NB: these have changed from previous versions of the unit tests # because it looks like appsessions created through the API now have an (empty) input samples list by default # TODO can't test this easily since self-created ssn don't have inputs. Add POST properties for ssns, and manually add an 'Input.Test' property, then test for it? - + def testGetAppSessionInputsByIdWithQp(self): props = self.api.getAppSessionInputsById(self.ssn.Id, qp({'Limit':1})) self.assertEqual(len(props), 1) @@ -1531,11 +1536,11 @@ def testSetAppSessionState_UpdatedStatus(self): ssn = self.api.setAppSessionState(self.ssn.Id, status, statusSummary) self.assertEqual(ssn.Status, status) self.assertEqual(ssn.StatusSummary, statusSummary) - + def testSetAppSessionStateToComplete(self): status = 'Complete' statusSummary = 'things are looking good' - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test setAppSessionState to " + status, "test setAppSessionState to " + status, appSessionId="") ssn = self.api.setAppSessionState(ar.AppSession.Id, status, statusSummary) self.assertEqual(ssn.Status, status) @@ -1544,16 +1549,16 @@ def testSetAppSessionStateToComplete(self): def testSetAppSessionStateToNeedsAttention(self): status = 'NeedsAttention' statusSummary = 'things are looking shaky' - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test setAppSessionState to " + status, "test setAppSessionState to " + status, appSessionId="") ssn = self.api.setAppSessionState(ar.AppSession.Id, status, statusSummary) self.assertEqual(ssn.Status, status) self.assertEqual(ssn.StatusSummary, statusSummary) - + def testSetAppSessionStateToTimedOut(self): status = 'TimedOut' statusSummary = 'things are falling behind' - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test setAppSessionState to " + status, "test setAppSessionState to " + status, appSessionId="") ssn = self.api.setAppSessionState(ar.AppSession.Id, status, statusSummary) self.assertEqual(ssn.Status, status) @@ -1562,93 +1567,93 @@ def testSetAppSessionStateToTimedOut(self): def testSetAppSessionStateToAborted(self): status = 'Aborted' statusSummary = 'things are looking bad' - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test setAppSessionState to " + status, "test setAppSessionState to " + status, appSessionId="") ssn = self.api.setAppSessionState(ar.AppSession.Id, status, statusSummary) self.assertEqual(ssn.Status, status) self.assertEqual(ssn.StatusSummary, statusSummary) - + def testSetAppSessionState_StatusException(self): status = 'PrettyMuchWorkingKindaSorta' statusSummary = 'tests, what tests' with self.assertRaises(AppSessionException): ssn = self.api.setAppSessionState(self.ssn.Id, status, statusSummary) - def test__deserializeObject__Project(self): + def test__deserializeObject__Project(self): type = 'Project' dct = { "HrefSamples": "testurl", "Gibberish": "more Gibberish" } - new_obj = self.api.__deserializeObject__(dct, type) + new_obj = self.api.__deserializeObject__(dct, type) self.assertEqual(new_obj.HrefSamples, "testurl") with self.assertRaises(AttributeError): self.assertEqual(new_obj.Gibberish, "more Gibberish") - + def test__deserializeObject__Sample(self): type = 'Sample' dct = { "SampleNumber": "123", "Gibberish": "more Gibberish" } - new_obj = self.api.__deserializeObject__(dct, type) + new_obj = self.api.__deserializeObject__(dct, type) self.assertEqual(new_obj.SampleNumber, 123) with self.assertRaises(AttributeError): self.assertEqual(new_obj.Gibberish, "more Gibberish") - + def test__deserializeObject__AppResult(self): type = 'AppResult' dct = { "Description": "Fuzzy", "Gibberish": "more Gibberish" } - new_obj = self.api.__deserializeObject__(dct, type) + new_obj = self.api.__deserializeObject__(dct, type) self.assertEqual(new_obj.Description, "Fuzzy") with self.assertRaises(AttributeError): self.assertEqual(new_obj.Gibberish, "more Gibberish") - + def test__deserializeObject__Other(self): type = 'Other' dct = { "Description": "Fuzzy", "Gibberish": "more Gibberish" } - new_obj = self.api.__deserializeObject__(dct, type) - self.assertEqual(new_obj, dct) - + new_obj = self.api.__deserializeObject__(dct, type) + self.assertEqual(new_obj, dct) + class TestAPICoverageMethods(TestCase): ''' Tests API Coverage object methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') - + def testGetIntervalCoverage(self): cov = self.api.getIntervalCoverage( Id = tconst['bam_file_id'], Chrom = tconst['bam_cov_chr_name'], StartPos = tconst['bam_cov_start_coord'], - EndPos = tconst['bam_cov_end_coord']) + EndPos = tconst['bam_cov_end_coord']) self.assertEqual(cov.Chrom, tconst['bam_cov_chr_name']) self.assertEqual(cov.StartPos, int(tconst['bam_cov_start_coord'])) - self.assertEqual(cov.EndPos, int(tconst['bam_cov_end_coord'])) + self.assertEqual(cov.EndPos, int(tconst['bam_cov_end_coord'])) def testGetCoverageMetaInfo(self): cov_meta = self.api.getCoverageMetaInfo( Id = tconst['bam_file_id'], Chrom = tconst['bam_cov_chr_name']) self.assertTrue(hasattr(cov_meta, 'MaxCoverage')) - + class TestAPIVariantMethods(TestCase): ''' Tests API Variant object methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') - - def testFilterVariantSet(self): + + def testFilterVariantSet(self): vars = self.api.filterVariantSet( - Id = tconst['vcf_file_id'], + Id = tconst['vcf_file_id'], Chrom = tconst['vcf_chr_name'], StartPos = tconst['vcf_start_coord'], - EndPos = tconst['vcf_end_coord'], ) + EndPos = tconst['vcf_end_coord'], ) self.assertEqual(vars[0].CHROM, tconst['vcf_chr_name']) - + def testFilterVariantWithQp(self): vars = self.api.filterVariantSet( - Id = tconst['vcf_file_id'], + Id = tconst['vcf_file_id'], Chrom = tconst['vcf_chr_name'], StartPos = tconst['vcf_start_coord'], EndPos = tconst['vcf_end_coord'], @@ -1656,8 +1661,8 @@ def testFilterVariantWithQp(self): queryPars = qp({'Limit':1}) ) self.assertEqual(vars[0].CHROM, tconst['vcf_chr_name']) self.assertEqual(len(vars), 1) - - def testFilterVariantReturnVCFString(self): + + def testFilterVariantReturnVCFString(self): with self.assertRaises(NotImplementedError): # for now... vars = self.api.filterVariantSet( Id = tconst['vcf_file_id'], @@ -1665,26 +1670,26 @@ def testFilterVariantReturnVCFString(self): StartPos = tconst['vcf_start_coord'], EndPos = tconst['vcf_end_coord'], Format = 'vcf') - #self.assertEqual(type(vars), str) - - def testGetVariantMeta(self): + #self.assertEqual(type(vars), str) + + def testGetVariantMeta(self): hdr = self.api.getVariantMetadata(tconst['vcf_file_id']) self.assertTrue(hasattr(hdr, 'Metadata')) - def testGetVariantMetaReturnVCFString(self): + def testGetVariantMetaReturnVCFString(self): with self.assertRaises(NotImplementedError): # for now... - hdr = self.api.getVariantMetadata(tconst['vcf_file_id'], Format='vcf') + hdr = self.api.getVariantMetadata(tconst['vcf_file_id'], Format='vcf') #self.assertEqual(type(hdr), str) - + class TestAPICredentialsMethods(TestCase): ''' Tests API object credentials methods - ''' - def setUp(self): + ''' + def setUp(self): self.profile = 'unit_tests' self.api = BaseSpaceAPI(profile=self.profile) - - def test_setCredentials_AllFromProfile(self): + + def test_setCredentials_AllFromProfile(self): creds = self.api._setCredentials(clientKey=None, clientSecret=None, apiServer=None, appSessionId='', apiVersion=self.api.version, accessToken='', profile=self.profile) @@ -1696,7 +1701,7 @@ def test_setCredentials_AllFromProfile(self): # self.assertEqual(creds['appSessionId'], self.api.appSessionId) self.assertEqual(creds['accessToken'], self.api.getAccessToken()) - def test_setCredentials_AllFromConstructor(self): + def test_setCredentials_AllFromConstructor(self): creds = self.api._setCredentials(clientKey='test_key', clientSecret='test_secret', apiServer='https://www.test.server.com', apiVersion='test_version', appSessionId='test_ssn', accessToken='test_token', profile=self.profile) @@ -1711,10 +1716,10 @@ def test_setCredentials_AllFromConstructor(self): def test_setCredentials_MissingConfigCredsException(self): # Danger: if this test fails unexpectedly, the config file may not be renamed back to the original name # 1) mv current .basespacepy.cfg, 2) create new with new content, - # 3) run test, 4) erase new, 5) mv current back + # 3) run test, 4) erase new, 5) mv current back cfg = os.path.expanduser('~/.basespace/unit_tests.cfg') tmp_cfg = cfg + '.unittesting.donotdelete' - shutil.move(cfg, tmp_cfg) + shutil.move(cfg, tmp_cfg) new_cfg_content = ("[" + self.profile + "]\n" "accessToken=test\n" "appSessionId=test\n") @@ -1733,24 +1738,24 @@ def test__setCredentials_DefaultsForOptionalArgs(self): # 3) run test, 4) erase new, 5) mv current back cfg = os.path.expanduser('~/.basespace/unit_tests.cfg') tmp_cfg = cfg + '.unittesting.donotdelete' - shutil.move(cfg, tmp_cfg) + shutil.move(cfg, tmp_cfg) new_cfg_content = ("[DEFAULT]\n" "clientKey=test\n" - "clientSecret=test\n" + "clientSecret=test\n" "apiServer=test\n" "apiVersion=test\n" "accessToken=test\n") with open(cfg, "w") as f: - f.write(new_cfg_content) + f.write(new_cfg_content) creds = self.api._setCredentials(clientKey=None, clientSecret=None, apiServer=None, apiVersion=self.api.version, appSessionId='', accessToken='', profile=self.profile) self.assertEqual(creds['appSessionId'], '') self.assertEqual(creds['accessToken'], 'test') os.remove(cfg) - shutil.move(tmp_cfg, cfg) + shutil.move(tmp_cfg, cfg) - def test__getLocalCredentials(self): + def test__getLocalCredentials(self): creds = self.api._getLocalCredentials(profile='unit_tests') self.assertEqual('name' in creds, True) # self.assertEqual('clientKey' in creds, True) @@ -1770,27 +1775,27 @@ def test__getLocalCredentials_DefaultProfile(self): # self.assertEqual('appSessionId' in creds, True) self.assertEqual('accessToken' in creds, True) - def test__getLocalCredentials_MissingProfile(self): + def test__getLocalCredentials_MissingProfile(self): with self.assertRaises(CredentialsException): - creds = self.api._getLocalCredentials(profile="SuperCallaFragaListic AppTastic") + creds = self.api._getLocalCredentials(profile="SuperCallaFragaListic AppTastic") class TestAPIGenomeMethods(TestCase): ''' Tests API object Genome methods - ''' - def setUp(self): + ''' + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') def testGetAvailableGenomes(self): - genomes = self.api.getAvailableGenomes() + genomes = self.api.getAvailableGenomes() #self.assertIsInstance(g[0], GenomeV1.GenomeV1) self.assertIsInstance(int(genomes[0].Id), int) - + def testGetAvailableGenomesWithQp(self): genomes = self.api.getAvailableGenomes(qp({'Limit':200})) genome = next(gen for gen in genomes if gen.Id == tconst['genome_id']) - self.assertTrue(genome.Id, tconst['genome_id']) - + self.assertTrue(genome.Id, tconst['genome_id']) + def testGetGenomeById(self): g = self.api.getGenomeById(tconst['genome_id']) self.assertEqual(g.Id, tconst['genome_id']) @@ -1799,16 +1804,16 @@ class TestAPIUtilityMethods(TestCase): ''' Tests utility methods of the API object ''' - def setUp(self): + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') - + def test_validateQueryParametersDefault(self): self.assertEqual(self.api._validateQueryParameters(None), {}) - + def test_validateQueryParameters(self): queryPars = {'Limit':10} self.assertEqual(self.api._validateQueryParameters( qp(queryPars) ), queryPars) - + def test_validateQueryParametersException(self): with self.assertRaises(QueryParameterException): self.api._validateQueryParameters({'Limit':10}) @@ -1817,12 +1822,12 @@ class TestQueryParametersMethods(TestCase): ''' Tests QueryParameters methods ''' - def testGetParameterDictAndValidate(self): + def testGetParameterDictAndValidate(self): queryp = qp({'Limit':1}, ['Limit']) - passed = queryp.getParameterDict() + passed = queryp.getParameterDict() self.assertEqual(passed, {'Limit':1}) self.assertEqual(queryp.validate(), None) - + def testNoDictException(self): with self.assertRaises(QueryParameterException): queryp = qp('test') @@ -1831,14 +1836,14 @@ def testValidateMissingRequiredParameterException(self): queryp = qp({'Limit':1}, ['I am required']) with self.assertRaises(UndefinedParameterException): queryp.validate() - + def testValidateUnknownParameterException(self): queryp = qp({'Crazy New Parameter':66}) with self.assertRaises(UnknownParameterException): queryp.validate() - + def testValidateIllegalValueForKnownQpKeyException(self): - queryp = qp({'SortBy': 'abc'}) + queryp = qp({'SortBy': 'abc'}) with self.assertRaises(IllegalParameterException): queryp.validate() @@ -1860,40 +1865,40 @@ class TestAPIOAuthMethods(TestCase): ''' Tests API Oauth methods ''' - def setUp(self): + def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') - + def testGetAccess_Device(self): proj = self.api.getProjectById(tconst['project_id']) resp = self.api.getAccess(proj, accessType='browse') self.assertTrue('device_code' in resp) - def testGetAccess_DeviceModelNotSupportedException(self): + def testGetAccess_DeviceModelNotSupportedException(self): with self.assertRaises(ModelNotSupportedException): - self.api.getAccess("test") - + self.api.getAccess("test") + def testGetAccess_Web(self): proj = self.api.getProjectById(tconst['project_id']) url = self.api.getAccess(proj, accessType='browse', web=True, redirectURL='http://www.basespacepy.tv', state='working') self.assertTrue(url.startswith('http')) - + def testGetVerificationCode(self): resp = self.api.getVerificationCode('browse project ' + tconst['project_id']) self.assertTrue('device_code' in resp) - + def testGetWebVerificationCode(self): url = self.api.getWebVerificationCode('browse project ' + tconst['project_id'], redirectURL='http://www.basespacepy.tv') self.assertTrue(url.startswith('http')) - self.assertTrue('state=' in url) - + self.assertTrue('state=' in url) + def testGetWebVerificationCode_WithStateParam(self): url = self.api.getWebVerificationCode('browse project ' + tconst['project_id'], redirectURL='http://www.basespacepy.tv', state='working') self.assertTrue(url.startswith('http')) self.assertTrue('state=working' in url) - + def testObtainAccessToken_DeviceApp(self): resp = self.api.getVerificationCode('browse project ' + tconst['project_id']) - webbrowser.open(resp['verification_with_code_uri']) + webbrowser.open(resp['verification_with_code_uri']) time.sleep(25) # wait for user to accept oauth request self.assertTrue(isinstance(self.api.obtainAccessToken(resp['device_code']), str)) @@ -1901,15 +1906,15 @@ def testObtainAccessToken_DeviceApp(self): def testObtainAccessToken_WebApp(self): with self.assertRaises(Exception): self.api.obtainAccessToken('123456', grantType='authorization_code', redirect_uri='http://www.basespacepy.tv') - - def testObtainAccessToken_WebAppRedirectURIException(self): + + def testObtainAccessToken_WebAppRedirectURIException(self): with self.assertRaises(OAuthException): self.api.obtainAccessToken('123456', grantType='authorization_code', redirect_uri=None) - + def testUpdatePrivileges_DeviceApp(self): resp = self.api.getVerificationCode('browse project ' + tconst['project_id']) - webbrowser.open(resp['verification_with_code_uri']) + webbrowser.open(resp['verification_with_code_uri']) time.sleep(25) # wait for user to accept oauth request origToken = self.api.getAccessToken() self.api.updatePrivileges(resp['device_code']) @@ -1919,15 +1924,15 @@ def testUpdatePrivileges_DeviceApp(self): @skip("Not sure how to test, since must parse auth code from redirect url - use django.test assertRedirects()?") def testUpdatePrivileges_WebApp(self): with self.assertRaises(Exception): - self.api.updatePrivileges('123456', grantType='authorization_code', redirect_uri='http://www.basespacepy.tv') + self.api.updatePrivileges('123456', grantType='authorization_code', redirect_uri='http://www.basespacepy.tv') class TestBaseSpaceAPIMethods(TestCase): ''' Tests BaseSpace API constructor and attributes; all methods tested in other testcases ''' def setUp(self): - self.api = BaseSpaceAPI(profile='unit_tests') - + self.api = BaseSpaceAPI(profile='unit_tests') + def test__init__(self): creds = self.api._getLocalCredentials(profile='unit_tests') # self.assertEqual(creds['appSessionId'], self.api.appSessionId) @@ -1943,9 +1948,9 @@ class TestBaseAPIMethods(TestCase): Tests Base API methods ''' def setUp(self): - api = BaseSpaceAPI(profile='unit_tests') + api = BaseSpaceAPI(profile='unit_tests') self.bapi = BaseAPI(api.getAccessToken(), api.apiClient.apiServerAndVersion) - + def test__init__(self): accessToken = "123" apiServerAndVersion = "http://api.tv" @@ -1957,8 +1962,8 @@ def test__init__(self): def test__singleRequest__(self): # get current user - resourcePath = '/users/current' - method = 'GET' + resourcePath = '/users/current' + method = 'GET' queryParams = {} headerParams = {} user = self.bapi.__singleRequest__(UserResponse.UserResponse, resourcePath, method, queryParams, headerParams) @@ -1966,42 +1971,42 @@ def test__singleRequest__(self): def test__singleRequest__WithPostData(self): # create a project - resourcePath = '/projects/' + resourcePath = '/projects/' method = 'POST' queryParams = {} headerParams = {} - postData = { 'Name': tconst['create_project_name'] } - proj = self.bapi.__singleRequest__(ProjectResponse.ProjectResponse, + postData = { 'Name': tconst['create_project_name'] } + proj = self.bapi.__singleRequest__(ProjectResponse.ProjectResponse, resourcePath, method, queryParams, headerParams, postData=postData) self.assertEqual(proj.Name, tconst['create_project_name']) - + def test__singleRequest__WithForcePost(self): # initiate a multipart upload -- requires a POST with no post data ('force post') - api = BaseSpaceAPI(profile='unit_tests') - proj = api.createProject(tconst['create_project_name']) - ar = proj.createAppResult(api, "test __singleResult__WithForcePost", "test __singleResult__WithForcePost", appSessionId="") - resourcePath = '/appresults/{Id}/files' + api = BaseSpaceAPI(profile='unit_tests') + proj = api.createProject(tconst['create_project_name']) + ar = proj.createAppResult(api, "test __singleResult__WithForcePost", "test __singleResult__WithForcePost", appSessionId="") + resourcePath = '/appresults/{Id}/files' method = 'POST' resourcePath = resourcePath.replace('{Id}', ar.Id) queryParams = {} queryParams['name'] = "test file name" queryParams['directory'] = "test directory" - queryParams['multipart'] = 'true' + queryParams['multipart'] = 'true' headerParams = {} - headerParams['Content-Type'] = 'text/plain' - postData = None + headerParams['Content-Type'] = 'text/plain' + postData = None file = self.bapi.__singleRequest__(FileResponse.FileResponse, resourcePath, method, queryParams, headerParams, postData=postData, forcePost=1) - self.assertTrue(hasattr(file, 'Id'), 'Successful force post should return file object with Id attribute here') + self.assertTrue(hasattr(file, 'Id'), 'Successful force post should return file object with Id attribute here') @skip("Not sure how to test this, requires no response from api server") def test__singleRequest__NoneResponseException(self): pass - + def test__singleRequest__ErrorResponseException(self): # malformed resoucePath, BadRequest Error and Message in response - resourcePath = '/users/curren' - method = 'GET' + resourcePath = '/users/curren' + method = 'GET' queryParams = {} headerParams = {} with self.assertRaises(ServerResponseException): @@ -2009,8 +2014,8 @@ def test__singleRequest__ErrorResponseException(self): def test__singleRequest__UnrecognizedPathResponseException(self): # malformed resoucePath, Message in response is 'not recognized path' (no error code) - resourcePath = '/users/current/run' - method = 'GET' + resourcePath = '/users/current/run' + method = 'GET' queryParams = {} headerParams = {} with self.assertRaises(ServerResponseException): @@ -2018,8 +2023,8 @@ def test__singleRequest__UnrecognizedPathResponseException(self): def test__listRequest__(self): # get current user - resourcePath = '/users/current/runs' - method = 'GET' + resourcePath = '/users/current/runs' + method = 'GET' queryParams = {} headerParams = {} runs = self.bapi.__listRequest__(Run.Run, resourcePath, method, queryParams, headerParams) @@ -2029,14 +2034,14 @@ def test__listRequest__(self): @skip("Not sure how to test this, requires no response from api server") def test__listRequest__NoneResponseException(self): pass - + def test__listRequest__ErrorResponseException(self): # Unauthorized - use nonsense acccess token - api = BaseSpaceAPI(profile='unit_tests') + api = BaseSpaceAPI(profile='unit_tests') bapi = BaseAPI(AccessToken="123123123123123123", apiServerAndVersion=api.apiClient.apiServerAndVersion) - resourcePath = '/users/current/uns' - method = 'GET' + resourcePath = '/users/current/uns' + method = 'GET' queryParams = {} headerParams = {} with self.assertRaises(ServerResponseException): @@ -2044,8 +2049,8 @@ def test__listRequest__ErrorResponseException(self): def test__listRequest__UnrecognizedPathResponseException(self): # malformed resoucePath, not recognized path message - resourcePath = '/users/current/uns' - method = 'GET' + resourcePath = '/users/current/uns' + method = 'GET' queryParams = {} headerParams = {} with self.assertRaises(ServerResponseException): @@ -2056,7 +2061,7 @@ def test__makeCurlRequest__(self): api = BaseSpaceAPI(profile='unit_tests') scope = 'browse project ' + tconst['project_id'] postData = [('client_id', api.key), ('scope', scope),('response_type', 'device_code')] - resp = self.bapi.__makeCurlRequest__(postData, api.apiClient.apiServerAndVersion + deviceURL) + resp = self.bapi.__makeCurlRequest__(postData, api.apiClient.apiServerAndVersion + deviceURL) self.assertTrue('device_code' in resp) @skip("Not sure how to test this, requires no response from api server") @@ -2070,19 +2075,19 @@ def test__makeCurlRequest__ServerErrorException(self): scope = 'browse project ' + tconst['project_id'] postData = [('client_id', 'gibberish'), ('scope', scope),('response_type', 'device_code')] with self.assertRaises(ServerResponseException): - self.bapi.__makeCurlRequest__(postData, api.apiClient.apiServerAndVersion + deviceURL) + self.bapi.__makeCurlRequest__(postData, api.apiClient.apiServerAndVersion + deviceURL) def testGetTimeout(self): self.assertEqual(self.bapi.getTimeout(), 10) - + def testSetTimeout(self): - self.bapi.setTimeout(20) + self.bapi.setTimeout(20) self.assertEqual(self.bapi.apiClient.timeout, 20) - + def testGetAccessToken(self): api = BaseSpaceAPI(profile='unit_tests') self.assertEqual(self.bapi.getAccessToken(), api.apiClient.apiKey) - + def testSetAccessToken(self): self.bapi.setAccessToken("abc") self.assertEqual(self.bapi.getAccessToken(), "abc") @@ -2093,197 +2098,197 @@ class TestAPIClientMethods(TestCase): ''' def setUp(self): self.api = BaseSpaceAPI(profile='unit_tests') - self.apiClient = APIClient(self.api.apiClient.apiKey, self.api.apiClient.apiServerAndVersion) - + self.apiClient = APIClient(self.api.apiClient.apiKey, self.api.apiClient.apiServerAndVersion) + def test__init__(self): accessToken = "abc" apiServerAndVersion = "http://basesinspaces.tv" timeout = 20 apiClient = APIClient(AccessToken=accessToken, apiServerAndVersion=apiServerAndVersion, timeout=timeout) - self.assertEqual(accessToken, apiClient.apiKey) + self.assertEqual(accessToken, apiClient.apiKey) self.assertEqual(apiServerAndVersion, apiClient.apiServerAndVersion) self.assertEqual(timeout, apiClient.timeout) def test__forcePostCall__(self): # initiate a multipart upload -- requires a POST with no post data ('force post') - # all method params are required for success in this example - resourcePath, headerParams, and postData(queryParams - proj = self.api.createProject(tconst['create_project_name']) - ar = proj.createAppResult(self.api, "test__forcePostCall__", "test__forcePostCall__", appSessionId="") - + # all method params are required for success in this example - resourcePath, headerParams, and postData(queryParams + proj = self.api.createProject(tconst['create_project_name']) + ar = proj.createAppResult(self.api, "test__forcePostCall__", "test__forcePostCall__", appSessionId="") + resourcePath = '/appresults/{Id}/files' resourcePath = resourcePath.replace('{Id}', ar.Id) queryParams = {} queryParams['name'] = "test file name" queryParams['directory'] = "test directory" - queryParams['multipart'] = 'true' + queryParams['multipart'] = 'true' headerParams = {} headerParams['Content-Type'] = 'text/plain' # normally added by callAPI() - headerParams['Authorization'] = 'Bearer ' + self.apiClient.apiKey + headerParams['Authorization'] = 'Bearer ' + self.apiClient.apiKey - jsonResp = self.apiClient.__forcePostCall__(resourcePath=self.apiClient.apiServerAndVersion + resourcePath, postData=queryParams, headers=headerParams) + jsonResp = self.apiClient.__forcePostCall__(resourcePath=self.apiClient.apiServerAndVersion + resourcePath, postData=queryParams, headers=headerParams) dictResp = json.loads(jsonResp) - self.assertTrue('Response' in dictResp, 'Successful force post should return json with Response attribute: ' + str(dictResp)) - self.assertTrue('Id' in dictResp['Response'], 'Successful force post should return json with Response with Id attribute: ' + str(dictResp)) + self.assertTrue('Response' in dictResp, 'Successful force post should return json with Response attribute: ' + str(dictResp)) + self.assertTrue('Id' in dictResp['Response'], 'Successful force post should return json with Response with Id attribute: ' + str(dictResp)) def test__putCall__(self): # upload a part of a multipart upload (the only PUT call in BaseSpacePy, for now) testDir = "test__putCall__" - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "test__putCall__", "test__putCall__", appSessionId="") file = self.api.__initiateMultipartFileUpload__( resourceType = 'appresults', resourceId = ar.Id, - fileName = os.path.basename(tconst['file_small_upload']), + fileName = os.path.basename(tconst['file_small_upload']), directory = testDir, contentType = tconst['file_small_upload_content_type']) with open(tconst['file_small_upload']) as fp: out = fp.read() md5 = hashlib.md5(out).digest().encode('base64') - + method = 'PUT' resourcePath = '/files/{Id}/parts/{partNumber}' resourcePath = resourcePath.replace('{Id}', file.Id) - resourcePath = resourcePath.replace('{partNumber}', str(1)) + resourcePath = resourcePath.replace('{partNumber}', str(1)) headerParams = {'Content-MD5': md5} data = tconst['file_small_upload_contents'] putResp = self.apiClient.__putCall__(resourcePath=self.apiClient.apiServerAndVersion + resourcePath, headers=headerParams, data=data) - #print "RESPONSE is: " + putResp + #print("RESPONSE is: " + putResp) jsonResp = putResp.split()[-1] # normally done in callAPI() dictResp = json.loads(jsonResp) - self.assertTrue('Response' in dictResp, 'Successful force post should return json with Response attribute: ' + str(dictResp)) - self.assertTrue('ETag' in dictResp['Response'], 'Successful force post should return json with Response with Id attribute: ' + str(dictResp)) + self.assertTrue('Response' in dictResp, 'Successful force post should return json with Response attribute: ' + str(dictResp)) + self.assertTrue('ETag' in dictResp['Response'], 'Successful force post should return json with Response with Id attribute: ' + str(dictResp)) - def testCallAPI_GET(self): - # get current user uses GET - resourcePath = '/users/current' - method = 'GET' + def testCallAPI_GET(self): + # get current user uses GET + resourcePath = '/users/current' + method = 'GET' queryParams = {} #headerParams = {} dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=None)#, headerParams=None, forcePost=False) - self.assertTrue('Response' in dictResp, 'response is: ' + str(dictResp)) - self.assertTrue('Id' in dictResp['Response']) - + self.assertTrue('Response' in dictResp, 'response is: ' + str(dictResp)) + self.assertTrue('Id' in dictResp['Response']) + @skip('There are no GET calls in the BaseSpace API that require headerParams') def testCallAPI_GETwithHeaderParams(self): pass - + def testCallAPI_POST(self): # create a project uses POST - resourcePath = '/projects/' + resourcePath = '/projects/' method = 'POST' queryParams = {} #headerParams = {} postData = {} - postData['Name'] = tconst['create_project_name'] + postData['Name'] = tconst['create_project_name'] dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=postData)#, headerParams=None, forcePost=False) self.assertTrue('Response' in dictResp) self.assertTrue('Id' in dictResp['Response']) def testCallAPI_POSTwithHeaderAndQueryParams(self): - # single part file upload uses POST with required qp and hdrs - proj = self.api.createProject(tconst['create_project_name']) - ar = proj.createAppResult(self.api, "test upload", "test upload", appSessionId="") + # single part file upload uses POST with required qp and hdrs + proj = self.api.createProject(tconst['create_project_name']) + ar = proj.createAppResult(self.api, "test upload", "test upload", appSessionId="") testDir = "testCallAPI_POSTwithHeaderAndQueryParams" fileName = os.path.basename(tconst['file_small_upload']) localPath=tconst['file_small_upload'] - + method = 'POST' - resourcePath = '/appresults/{Id}/files' + resourcePath = '/appresults/{Id}/files' resourcePath = resourcePath.replace('{Id}', ar.Id) queryParams = {} queryParams['name'] = fileName - queryParams['directory'] = testDir + queryParams['directory'] = testDir headerParams = {} - headerParams['Content-Type'] = tconst['file_small_upload_content_type'] + headerParams['Content-Type'] = tconst['file_small_upload_content_type'] postData = open(localPath).read() - dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=postData, headerParams=headerParams)#, forcePost=False) + dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=postData, headerParams=headerParams)#, forcePost=False) self.assertTrue('Response' in dictResp) - self.assertTrue('Id' in dictResp['Response']) + self.assertTrue('Id' in dictResp['Response']) self.assertEqual(dictResp['Response']['Path'], os.path.join(testDir, fileName)) def testCallAPI_ForcePOST(self): # initiate a multipart upload -- requires a POST with no post data ('force post') - # all method params are required for success in this example - resourcePath, headerParams, and postData(queryParams - proj = self.api.createProject(tconst['create_project_name']) - ar = proj.createAppResult(self.api, "test__forcePostCall__", "test__forcePostCall__", appSessionId="") - + # all method params are required for success in this example - resourcePath, headerParams, and postData(queryParams + proj = self.api.createProject(tconst['create_project_name']) + ar = proj.createAppResult(self.api, "test__forcePostCall__", "test__forcePostCall__", appSessionId="") + method = 'POST' resourcePath = '/appresults/{Id}/files' resourcePath = resourcePath.replace('{Id}', ar.Id) queryParams = {} queryParams['name'] = "test file name" queryParams['directory'] = "test directory" - queryParams['multipart'] = 'true' + queryParams['multipart'] = 'true' headerParams = {} headerParams['Content-Type'] = 'text/plain' postData = None dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=postData, headerParams=headerParams, forcePost=True) - self.assertTrue('Response' in dictResp, 'Successful force post should return json with Response attribute: ' + str(dictResp)) - self.assertTrue('Id' in dictResp['Response'], 'Successful force post should return json with Response with Id attribute: ' + str(dictResp)) + self.assertTrue('Response' in dictResp, 'Successful force post should return json with Response attribute: ' + str(dictResp)) + self.assertTrue('Id' in dictResp['Response'], 'Successful force post should return json with Response with Id attribute: ' + str(dictResp)) def testCallAPI_PUT(self): # upload a part of a multipart upload (the only PUT call in BaseSpacePy, for now) testDir = "testCallAPI_PUT" - proj = self.api.createProject(tconst['create_project_name']) + proj = self.api.createProject(tconst['create_project_name']) ar = proj.createAppResult(self.api, "testCallAPI_PUT", "testCallAPI_PUT", appSessionId="") file = self.api.__initiateMultipartFileUpload__( resourceType = 'appresults', resourceId = ar.Id, - fileName = os.path.basename(tconst['file_small_upload']), + fileName = os.path.basename(tconst['file_small_upload']), directory = testDir, contentType = tconst['file_small_upload_content_type']) with open(tconst['file_small_upload']) as fp: out = fp.read() md5 = hashlib.md5(out).digest().encode('base64') - + method = 'PUT' resourcePath = '/files/{Id}/parts/{partNumber}' resourcePath = resourcePath.replace('{Id}', file.Id) - resourcePath = resourcePath.replace('{partNumber}', str(1)) + resourcePath = resourcePath.replace('{partNumber}', str(1)) headerParams = {'Content-MD5': md5} queryParams = {} # not used for PUT calls data = tconst['file_small_upload_contents'] dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=data, headerParams=headerParams) - self.assertTrue('Response' in dictResp, 'Successful force post should return json with Response attribute: ' + str(dictResp)) - self.assertTrue('ETag' in dictResp['Response'], 'Successful force post should return json with Response with Id attribute: ' + str(dictResp)) + self.assertTrue('Response' in dictResp, 'Successful force post should return json with Response attribute: ' + str(dictResp)) + self.assertTrue('ETag' in dictResp['Response'], 'Successful force post should return json with Response with Id attribute: ' + str(dictResp)) - def testCallAPI_DELETE(self): + def testCallAPI_DELETE(self): method = 'DELETE' - resourcePath = '' - queryParams = {} + resourcePath = '' + queryParams = {} with self.assertRaises(NotImplementedError): dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=None) def testCallAPI_UnrecognizedRESTmethodException(self): method = 'TAKEOVERTHEWORLD' - resourcePath = '' - queryParams = {} + resourcePath = '' + queryParams = {} with self.assertRaises(RestMethodException): dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=None) def testCallAPI_HandleHttpError_ForGET(self): - # bad access token throws 401 Error and HTTPError exception by urllib2; get current user uses GET + # bad access token throws 401 Error and HTTPError exception by urllib2; get current user uses GET self.apiClient.apiKey = 'badtoken' - resourcePath = '/users/current' - method = 'GET' + resourcePath = '/users/current' + method = 'GET' queryParams = {} dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=None) - self.assertTrue('ResponseStatus' in dictResp, 'response is: ' + str(dictResp)) + self.assertTrue('ResponseStatus' in dictResp, 'response is: ' + str(dictResp)) self.assertTrue('ErrorCode' in dictResp['ResponseStatus']) self.assertTrue('Message' in dictResp['ResponseStatus']) self.assertTrue('Unrecognized access token' in dictResp['ResponseStatus']['Message']) def testCallAPI_HandleHttpError_ForPOST(self): - # bad access token throws 401 Error and HTTPError exception by urllib2; create a project uses POST - self.apiClient.apiKey = 'badtoken' - resourcePath = '/projects/' + # bad access token throws 401 Error and HTTPError exception by urllib2; create a project uses POST + self.apiClient.apiKey = 'badtoken' + resourcePath = '/projects/' method = 'POST' queryParams = {} postData = {} - postData['Name'] = tconst['create_project_name'] + postData['Name'] = tconst['create_project_name'] dictResp = self.apiClient.callAPI(resourcePath, method, queryParams, postData=postData) - self.assertTrue('ResponseStatus' in dictResp, 'response is: ' + str(dictResp)) + self.assertTrue('ResponseStatus' in dictResp, 'response is: ' + str(dictResp)) self.assertTrue('ErrorCode' in dictResp['ResponseStatus']) self.assertTrue('Message' in dictResp['ResponseStatus']) self.assertTrue('Unrecognized access token' in dictResp['ResponseStatus']['Message']) @@ -2297,7 +2302,7 @@ def testDeserialize_ClassObjClass_String(self): objClass = str out = self.apiClient.deserialize(obj, objClass) self.assertEqual(out, obj) - + def testDeserialize_ClassObjClass_Integer(self): obj = 123 objClass = int @@ -2315,7 +2320,7 @@ def testDeserialize_ClassObjClass_Float(self): objClass = float out = self.apiClient.deserialize(obj, objClass) self.assertEqual(out, obj) - + def testDeserialize_StringObjClass_String(self): obj = "test" objClass = 'str' @@ -2347,7 +2352,7 @@ def testDeserialize_ClassObjClass_Project(self): self.assertEqual(out.Id, "123") # not testing passing in an unknown class - + def testDeserialize_StringObjClass_Project(self): obj = {"Id":"123"} objClass = "Project" @@ -2362,23 +2367,23 @@ def testDeserialize_StringObjClass_File(self): # not testing passing in an unknown class - def testDeserialize_ClassObjClass_DynamicType(self): + def testDeserialize_ClassObjClass_DynamicType(self): obj = { 'ResponseStatus': 'test', 'Response': { # DynamicType - # MultiValueAppResultList - 'Type': 'appresult[]', + # MultiValueAppResultList + 'Type': 'appresult[]', 'DisplayedCount': 10, - }, + }, 'Notifications': '' } objClass = MultiValuePropertyResponse.MultiValuePropertyResponse out = self.apiClient.deserialize(obj, objClass) self.assertEqual(out.Response.DisplayedCount, 10) - # not testing passing in an unrecognized dynamic type - should warn - + # not testing passing in an unrecognized dynamic type - should warn + def testDeserialize_ClassObjClass_List(self): - obj = { 'CHROM': 'chr3', + obj = { 'CHROM': 'chr3', 'ID': ['1', '2', '3'] } # 'list' objClass = Variant.Variant out = self.apiClient.deserialize(obj, objClass) @@ -2388,12 +2393,12 @@ def testDeserialize_ClassObjClass_ListOfDynamicTypes(self): obj = { 'Items': [ # 'list', {'Type': 'string', 'Name': 'teststring'}, # PropertyString {'Type': 'project', 'Name': 'testproject'}, # PropertyProject - ], } + ], } objClass = PropertyList.PropertyList out = self.apiClient.deserialize(obj, objClass) self.assertEqual(out.Items[0].Name, 'teststring') self.assertEqual(out.Items[1].Name, 'testproject') - + def testDeserialize_ClassObjClass_ListOfLists(self): obj = { 'Items': [ #'listoflists', [ {'Key': 'testA1'}, {'Key': 'testA2'}], # PropertyMapKeyValues @@ -2405,13 +2410,13 @@ def testDeserialize_ClassObjClass_ListOfLists(self): self.assertEqual(out.Items[1][1].Key, 'testB2') def testDeserialize_ClassObjClass_Dict(self): - obj = { 'INFO': 'test' } # dict + obj = { 'INFO': 'test' } # dict objClass = Variant.Variant out = self.apiClient.deserialize(obj, objClass) self.assertEqual(out.INFO, 'test') - + def testDeserialize_ClassObjClass_Datetime(self): - obj = { 'DateCreated': '2013-10-03T19:40:26.0000000' } # datetime + obj = { 'DateCreated': '2013-10-03T19:40:26.0000000' } # datetime objClass = Run.Run out = self.apiClient.deserialize(obj, objClass) self.assertEqual(out.DateCreated.year, 2013) @@ -2429,7 +2434,7 @@ class TestBillingAPIMethods(TestCase): @skip('Test not written yet') def test__init__(self): pass - + class TestQueryParameterPurchasedProductMethods(TestCase): ''' Tests QueryParameterPurchasedProduct methods @@ -2439,12 +2444,12 @@ def test__init__(self): pass -#if __name__ == '__main__': +#if __name__ == '__main__': # main() # unittest.main() large_file_transfers = TestSuite([ TestLoader().loadTestsFromTestCase( TestAPIFileUploadMethods_LargeFiles ), TestLoader().loadTestsFromTestCase( TestAPIFileDownloadMethods_LargeFiles ), - TestLoader().loadTestsFromTestCase( TestMultipartFileTransferMethods ), ]) + TestLoader().loadTestsFromTestCase( TestMultipartFileTransferMethods ), ]) small_file_transfers = TestSuite([ TestLoader().loadTestsFromTestCase(TestFileDownloadMethods), @@ -2457,7 +2462,7 @@ def test__init__(self): TestLoader().loadTestsFromTestCase(TestUserMethods), TestLoader().loadTestsFromTestCase(TestAPIUserMethods), TestLoader().loadTestsFromTestCase(TestFileMethods), - TestLoader().loadTestsFromTestCase(TestAPIFileMethods), ]) + TestLoader().loadTestsFromTestCase(TestAPIFileMethods), ]) samples_appresults_projects = TestSuite([ TestLoader().loadTestsFromTestCase(TestSampleMethods), @@ -2502,20 +2507,20 @@ def test__init__(self): if(len(sys.argv) == 1): # to test all test cases: - tests.extend([ - small_file_transfers, - runs_users_files, + tests.extend([ + small_file_transfers, + runs_users_files, samples_appresults_projects, - appsessions, + appsessions, cred_genome_util_lists, - cov_variant, + cov_variant, basespaceapi_baseapi_apiclient, billing_qppp, ]) #tests.append(oauth) # these tests will open a web browser and clicking 'Accept' (also requires BaseSpace login) tests.append(large_file_transfers) # these tests may take tens of minutes to complete else: - # to test individual test cases: + # to test individual test cases: for t in sys.argv[1:]: tests.append( TestLoader().loadTestsFromTestCase( eval(t) ) ) TextTestRunner(verbosity=2).run( TestSuite(tests) ) From d88725e138ad93c540fbd0b78bd68fe7b066acbc Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Tue, 18 Oct 2016 14:57:40 -0700 Subject: [PATCH 05/22] cleanup --- test/unit_tests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/unit_tests.py b/test/unit_tests.py index eb2217c..94bfd54 100644 --- a/test/unit_tests.py +++ b/test/unit_tests.py @@ -219,8 +219,6 @@ def test__finalizeMultipartFileUpload__(self): contentType=tconst['file_small_upload_content_type']) with open(tconst['file_small_upload']) as fp: out = fp.read() - # md5 = hashlib.md5(out).digest().encode('base64') - # import pdb;pdb.set_trace() md5 = base64.b64encode(hashlib.md5(out.encode('utf-8')).digest()) response = self.api.__uploadMultipartUnit__( Id = file.Id, From e19f835a91b5f953b7af1d56cef0e3567efec180 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Tue, 18 Oct 2016 15:08:01 -0700 Subject: [PATCH 06/22] run test script with -e flag to catch errors --- runtests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtests.sh b/runtests.sh index 9481115..603d82f 100755 --- a/runtests.sh +++ b/runtests.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + mkdir ~/.basespace cp test/dotbasespace/*.bash ~/.basespace From b0fc15882631b11b2ed63dc432c6e17ce9aedb94 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Tue, 18 Oct 2016 15:14:08 -0700 Subject: [PATCH 07/22] test proper exit (should fail travis build) --- runtests.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runtests.sh b/runtests.sh index 603d82f..5a1a395 100755 --- a/runtests.sh +++ b/runtests.sh @@ -16,4 +16,8 @@ cat test/dotbasespace/unit_tests.cfg | sed "s/__ACCESS_TOKEN__/$ACCESS_TOKEN/" > cp ~/.basespace/unit_tests.cfg ~/.basespace/default.cfg -python test/unit_tests.py +# python test/unit_tests.py + +ls /jhlkjgfhlkjghdsdsdsds + +exit $? From a6ca107d6a2b36292d38f95062f4c578ca99ba2a Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Tue, 18 Oct 2016 15:16:05 -0700 Subject: [PATCH 08/22] test runner script should exit appropriately --- runtests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/runtests.sh b/runtests.sh index 5a1a395..54d70f9 100755 --- a/runtests.sh +++ b/runtests.sh @@ -16,8 +16,7 @@ cat test/dotbasespace/unit_tests.cfg | sed "s/__ACCESS_TOKEN__/$ACCESS_TOKEN/" > cp ~/.basespace/unit_tests.cfg ~/.basespace/default.cfg -# python test/unit_tests.py +python test/unit_tests.py -ls /jhlkjgfhlkjghdsdsdsds exit $? From 3c8b9117d18577d0d322c9b9fbf83547e498b105 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Tue, 18 Oct 2016 15:25:24 -0700 Subject: [PATCH 09/22] test suite exits with appropriate error code, for CI integration --- test/unit_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/unit_tests.py b/test/unit_tests.py index 94bfd54..2e46664 100644 --- a/test/unit_tests.py +++ b/test/unit_tests.py @@ -2521,4 +2521,6 @@ def test__init__(self): # to test individual test cases: for t in sys.argv[1:]: tests.append( TestLoader().loadTestsFromTestCase( eval(t) ) ) - TextTestRunner(verbosity=2).run( TestSuite(tests) ) + ret = TextTestRunner(verbosity=2).run( TestSuite(tests) ) + exitcode = not ret.wasSuccessful() + sys.exit(exitcode) From b6a38578f8790685f8a7b0c288c47def11978699 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Thu, 20 Oct 2016 14:28:09 -0700 Subject: [PATCH 10/22] unit tests in py2 really pass now (at least locally) --- src/BaseSpacePy/api/APIClient.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/BaseSpacePy/api/APIClient.py b/src/BaseSpacePy/api/APIClient.py index 5fb1be4..b8c56b8 100644 --- a/src/BaseSpacePy/api/APIClient.py +++ b/src/BaseSpacePy/api/APIClient.py @@ -137,13 +137,14 @@ def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None if not forcePost: if data and not len(data): data='\n' # temp fix, in case is no data in the file, to prevent post request from failing - data = data.encode('utf-8') + if isinstance(data, unicode): + data = data.encode('utf-8') request = urllib.request.Request(url=url, headers=headers, data=data)#,timeout=self.timeout) else: response = self.__forcePostCall__(forcePostUrl, sentQueryParams, headers) if method in ['PUT', 'DELETE']: if method == 'DELETE': - raise NotImplementedError("DELETE REST API calls aren't currently supported") + raise NotImplementedError('DELETE REST API calls aren\'t currently supported') response = self.__putCall__(url, headers, data) response = response.split()[-1] # discard upload status msg (from curl put?) else: From 1d5ab66b133c91ccf5d9d59e1a1a53a7edf56c93 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Thu, 20 Oct 2016 14:48:02 -0700 Subject: [PATCH 11/22] fix TestAPIFileUploadMethods_SmallFiles in py3 --- src/BaseSpacePy/api/APIClient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/BaseSpacePy/api/APIClient.py b/src/BaseSpacePy/api/APIClient.py index b8c56b8..94be7d3 100644 --- a/src/BaseSpacePy/api/APIClient.py +++ b/src/BaseSpacePy/api/APIClient.py @@ -137,7 +137,8 @@ def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None if not forcePost: if data and not len(data): data='\n' # temp fix, in case is no data in the file, to prevent post request from failing - if isinstance(data, unicode): + # if isinstance(data, unicode): + if data and six.PY3: data = data.encode('utf-8') request = urllib.request.Request(url=url, headers=headers, data=data)#,timeout=self.timeout) else: From 6b8d50f57244f3a350f9cd8cbe75cc4913ec0717 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Fri, 21 Oct 2016 11:20:36 -0700 Subject: [PATCH 12/22] add missing exception DownloadFailedException --- src/BaseSpacePy/api/BaseSpaceException.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/BaseSpacePy/api/BaseSpaceException.py b/src/BaseSpacePy/api/BaseSpaceException.py index 579cffc..1c132a4 100644 --- a/src/BaseSpacePy/api/BaseSpaceException.py +++ b/src/BaseSpacePy/api/BaseSpaceException.py @@ -16,31 +16,31 @@ def __init__(self, value,legal): self.parameter = str(value) + ' is not well-defined, legal options are ' + str(legal) def __str__(self): return repr(self.parameter) - + class WrongFiletypeException(Exception): def __init__(self, filetype): self.parameter = 'This data request is not available for file ' + str(filetype) def __str__(self): return repr(self.parameter) - + class ServerResponseException(Exception): def __init__(self, value): self.parameter = 'Error with API server response: ' + value def __str__(self): return repr(self.parameter) - + class ModelNotInitializedException(Exception): def __init__(self,value): self.parameter = 'The request cannot be completed as model has not been initialized - ' + value def __str__(self): return repr(self.parameter) - + class ByteRangeException(Exception): def __init__(self, value): self.parameter = 'Byte-range invalid: ' + value def __str__(self): return repr(self.parameter) - + class MultiProcessingTaskFailedException(Exception): def __init__(self, value): self.parameter = 'Multiprocessing task failed: ' + value @@ -88,4 +88,9 @@ def __init__(self, value): self.parameter = 'Problem with REST API method: ' + value def __str__(self): return repr(self.parameter) - + +class DownloadFailedException(Exception): + def __init__(self, value): + self.parameter = 'Download failed: ' + value + def __str__(self): + return repr(self.parameter) From 19c88241bb9fc0a96c5c2e6bc8f1b8ea80ebf196 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Fri, 21 Oct 2016 11:21:07 -0700 Subject: [PATCH 13/22] fix some broken unit tests by encoding response to utf-8 --- src/BaseSpacePy/api/APIClient.develop.py | 255 +++++++++++++++++++++ src/BaseSpacePy/api/APIClient.passes2.py | 270 +++++++++++++++++++++++ src/BaseSpacePy/api/APIClient.py | 3 +- 3 files changed, 526 insertions(+), 2 deletions(-) create mode 100644 src/BaseSpacePy/api/APIClient.develop.py create mode 100644 src/BaseSpacePy/api/APIClient.passes2.py diff --git a/src/BaseSpacePy/api/APIClient.develop.py b/src/BaseSpacePy/api/APIClient.develop.py new file mode 100644 index 0000000..52d32b4 --- /dev/null +++ b/src/BaseSpacePy/api/APIClient.develop.py @@ -0,0 +1,255 @@ + +import sys +import os +import re +import urllib +import urllib2 +import io +import cStringIO +import json +from subprocess import * +import subprocess +import dateutil.parser +from warnings import warn +from BaseSpacePy.model import * +from BaseSpacePy.api.BaseSpaceException import RestMethodException, ServerResponseException + + +class APIClient: + def __init__(self, AccessToken, apiServerAndVersion, userAgent=None, timeout=10): + ''' + Initialize the API instance + + :param AccessToken: an access token + :param apiServerAndVersion: the URL of the BaseSpace api server with api version + :param timeout: (optional) the timeout in seconds for each request made, default 10 + ''' + self.apiKey = AccessToken + self.apiServerAndVersion = apiServerAndVersion + self.userAgent = userAgent + self.timeout = timeout + + def __forcePostCall__(self, resourcePath, postData, headers): + ''' + For forcing a REST POST request (seems to be used when POSTing with no post data) + + :param resourcePath: the url to call, including server address and api version + :param postData: a dictionary of data to post + :param headers: a dictionary of header key/values to include in call + :returns: server response (a string containing json) + ''' + import requests + # this tries to clean up the output at the expense of letting the user know they're in an insecure context... + try: + requests.packages.urllib3.disable_warnings() + except: + pass + import logging + logging.getLogger("requests").setLevel(logging.WARNING) + encodedPost = urllib.urlencode(postData) + resourcePath = "%s?%s" % (resourcePath, encodedPost) + response = requests.post(resourcePath, data=json.dumps(postData), headers=headers) + return response.text + + def __putCall__(self, resourcePath, headers, data): + ''' + Performs a REST PUT call to the API server. + + :param resourcePath: the url to call, including server address and api version + :param headers: a dictionary of header key/values to include in call + :param transFile: the name of the file containing only data to be PUT + :returns: server response (a string containing upload status message (from curl?) followed by json response) + ''' + # headerPrep = [k + ':' + headers[k] for k in headers.keys()] + # cmd = 'curl -H "x-access-token:' + self.apiKey + '" -H "Content-MD5:' + headers['Content-MD5'].strip() +'" -T "'+ transFile +'" -X PUT ' + resourcePath + # p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) + # output = p.stdout.read() + # print output + # return output + import requests + put_headers = { + 'Content-MD5' : headers['Content-MD5'].strip(), + 'x-access-token': self.apiKey + } + put_val = requests.put(resourcePath, data, headers=put_headers) + if put_val.status_code != 200: + raise ServerResponseException("Multi-part upload: Server return code %s with error %s" % (put_val.status_code, put_val.reason)) + return put_val.text + + def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None, forcePost=False): + ''' + Call a REST API and return the server response. + + An access token header is automatically added. + If a Content-Type header isn't included, one will be added with 'application/json' (except for PUT and forcePost calls). + Query parameters with values of None aren't sent to the server. + Server errors are to be handled by the caller (returned response contains error codes/msgs). + + :param resourcePath: the url to call, not including server address and api version + :param method: REST method, including GET, POST (and forcePost, see below), and PUT (DELETE not yet supported) + :param queryParams: dictionary of query parameters to be added to url, except for forcePost where they are added as 'postData'; not used for PUT calls + :param postData: for POST calls, a dictionary to post; not used for forcePost calls; for PUT calls, name of file to put + :param headerParams: (optional) a dictionary of header data, default None + :param forcePost: (optional) 'force' a POST call using curl (instead of urllib), default False + + :raises RestMethodException: for unrecognized REST method + :raises ServerResponseException: for errors in parsing json response from server, and for urlerrors from the opening url + :returns: Server response deserialized to a python object (dict) + ''' + url = self.apiServerAndVersion + resourcePath + headers = {} + if self.userAgent: + headers['User-Agent'] = self.userAgent + if headerParams: + for param, value in headerParams.iteritems(): + headers[param] = value + # specify the content type + if not headers.has_key('Content-Type') and not method=='PUT' and not forcePost: + headers['Content-Type'] = 'application/json' + # include access token in header + headers['Authorization'] = 'Bearer ' + self.apiKey + + data = None + if method == 'GET': + if queryParams: + # Need to remove None values, these should not be sent + sentQueryParams = {} + for param, value in queryParams.iteritems(): + if value != None: + sentQueryParams[param] = value + url = url + '?' + urllib.urlencode(sentQueryParams) + request = urllib2.Request(url=url, headers=headers) + elif method in ['POST', 'PUT', 'DELETE']: + if queryParams: + # Need to remove None values, these should not be sent + sentQueryParams = {} + for param, value in queryParams.iteritems(): + if value != None: + sentQueryParams[param] = value + forcePostUrl = url + url = url + '?' + urllib.urlencode(sentQueryParams) + data = postData + if data: + if type(postData) not in [str, int, float, bool]: + data = json.dumps(postData) + if not forcePost: + if data and not len(data): + data='\n' # temp fix, in case is no data in the file, to prevent post request from failing + request = urllib2.Request(url=url, headers=headers, data=data)#,timeout=self.timeout) + else: + response = self.__forcePostCall__(forcePostUrl, sentQueryParams, headers) + if method in ['PUT', 'DELETE']: + if method == 'DELETE': + raise NotImplementedError('DELETE REST API calls aren\'t currently supported') + response = self.__putCall__(url, headers, data) + response = response.split()[-1] # discard upload status msg (from curl put?) + else: + raise RestMethodException('Method ' + method + ' is not recognized.') + + # Make the request + if not forcePost and not method in ['PUT', 'DELETE']: # the normal case + try: + response = urllib2.urlopen(request, timeout=self.timeout).read() + except urllib2.HTTPError as e: + response = e.read() # treat http error as a response (handle in caller) + except urllib2.URLError as e: + raise ServerResponseException('URLError: ' + str(e)) + try: + data = json.loads(response) + except ValueError as e: + raise ServerResponseException('Error decoding json in server response') + return data + + def deserialize(self, obj, objClass): + """ + Deserialize a JSON string into a BaseSpacePy object. + + :param obj: A dictionary (or object?) to be deserialized into a class (objClass); or a value to be passed into a new native python type (objClass) + :param objClass: A class object or native python type for the deserialized object, or a string of a class name or native python type. (eg, Project.Project, int, 'Project', 'int') + :returns: A deserialized object + """ + # Create an object class from objClass, if a string was passed in + # Avoid native python types 'file' + if type(objClass) == str: + try: + if (not str(objClass)=='File'): + objClass = eval(objClass.lower()) + else: + objClass = eval(objClass + '.' + objClass) + except NameError: # not a native type, must be model class + objClass = eval(objClass + '.' + objClass) + + # Create an instance of the object class + # If the instance is a native python type, return it + if objClass in [str, int, float, bool]: + return objClass(obj) + instance = objClass() + + # For every swaggerType in the instance that is also in the passed-in obj, + # set the instance value for native python types, + # or recursively deserialize class instances. + # For dynamic types, substitute real class after looking up 'Type' value. + # For lists, deserialize all members of a list, including lists of lists (though not list of list of list...). + # For datetimes, convert to a readable output string + for attr, attrType in instance.swaggerTypes.iteritems(): + if attr in obj: + value = obj[attr] + if attrType in ['str', 'int', 'float', 'bool']: + attrType = eval(attrType) + try: + value = attrType(value) + except UnicodeEncodeError: + value = unicode(value) + setattr(instance, attr, value) + elif attrType == 'DynamicType': + try: + model_name = instance._dynamicType[value['Type']] + except KeyError: + pass + # suppress this warning, which is caused by a bug in BaseSpace + #warn("Warning - unrecognized dynamic type: " + value['Type']) + else: + setattr(instance, attr, self.deserialize(value, model_name)) + elif 'list<' in attrType: + match = re.match('list<(.*)>', attrType) + subClass = match.group(1) + subValues = [] + + # lists of dynamic type + if subClass == 'DynamicType': + for subValue in value: + try: + new_type = instance._dynamicType[subValue['Type']] + except KeyError: + pass + # suppress this warning, which is caused by a bug in BaseSpace + #warn("Warning - unrecognized (list of) dynamic types: " + subValue['Type']) + else: + subValues.append(self.deserialize(subValue, new_type)) + setattr(instance, attr, subValues) + # typical lists + else: + for subValue in value: + subValues.append(self.deserialize(subValue, subClass)) + setattr(instance, attr, subValues) + # list of lists (e.g. map[] property type) + elif 'listoflists<' in attrType: + match = re.match('listoflists<(.*)>', attrType) + subClass = match.group(1) + outvals = [] + for outval in value: + invals = [] + for inval in outval: + invals.append(self.deserialize(inval, subClass)) + outvals.append(invals) + setattr(instance, attr, outvals) + + elif attrType=='dict': + setattr(instance, attr, value) + elif attrType=='datetime': + dt = dateutil.parser.parse(value) + setattr(instance, attr, dt) + else: + # recursive call with attribute type + setattr(instance, attr, self.deserialize(value, attrType)) + return instance diff --git a/src/BaseSpacePy/api/APIClient.passes2.py b/src/BaseSpacePy/api/APIClient.passes2.py new file mode 100644 index 0000000..abf1ef5 --- /dev/null +++ b/src/BaseSpacePy/api/APIClient.passes2.py @@ -0,0 +1,270 @@ + +import sys +import os +import re +import urllib +import urllib2 +import io +import cStringIO +import json +import logging +from subprocess import * +import subprocess +import dateutil.parser +from warnings import warn +from BaseSpacePy.model import * +from BaseSpacePy.api.BaseSpaceException import RestMethodException, ServerResponseException + +from six.moves import urllib as six_urllib +import six + +class APIClient: + def __init__(self, AccessToken, apiServerAndVersion, userAgent=None, timeout=10): + ''' + Initialize the API instance + + :param AccessToken: an access token + :param apiServerAndVersion: the URL of the BaseSpace api server with api version + :param timeout: (optional) the timeout in seconds for each request made, default 10 + ''' + self.apiKey = AccessToken + self.apiServerAndVersion = apiServerAndVersion + self.userAgent = userAgent + self.timeout = timeout + + def __forcePostCall__(self, resourcePath, postData, headers): + ''' + For forcing a REST POST request (seems to be used when POSTing with no post data) + + :param resourcePath: the url to call, including server address and api version + :param postData: a dictionary of data to post + :param headers: a dictionary of header key/values to include in call + :returns: server response (a string containing json) + ''' + import requests + # this tries to clean up the output at the expense of letting the user know they're in an insecure context... + try: + requests.packages.urllib3.disable_warnings() + except: + pass + import logging + logging.getLogger("requests").setLevel(logging.WARNING) + encodedPost = urllib.urlencode(postData) + # encodedPost = six_urllib.parse.urlencode(postData) + resourcePath = "%s?%s" % (resourcePath, encodedPost) + response = requests.post(resourcePath, data=json.dumps(postData), headers=headers) + return response.text + + def __putCall__(self, resourcePath, headers, data): + ''' + Performs a REST PUT call to the API server. + + :param resourcePath: the url to call, including server address and api version + :param headers: a dictionary of header key/values to include in call + :param transFile: the name of the file containing only data to be PUT + :returns: server response (a string containing upload status message (from curl?) followed by json response) + ''' + # headerPrep = [k + ':' + headers[k] for k in headers.keys()] + # cmd = 'curl -H "x-access-token:' + self.apiKey + '" -H "Content-MD5:' + headers['Content-MD5'].strip() +'" -T "'+ transFile +'" -X PUT ' + resourcePath + # p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) + # output = p.stdout.read() + # print output + # return output + import requests + put_headers = { + 'Content-MD5' : headers['Content-MD5'].strip(), + 'x-access-token': self.apiKey + } + put_val = requests.put(resourcePath, data, headers=put_headers) + if put_val.status_code != 200: + raise ServerResponseException("Multi-part upload: Server return code %s with error %s" % (put_val.status_code, put_val.reason)) + return put_val.text + + def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None, forcePost=False): + ''' + Call a REST API and return the server response. + + An access token header is automatically added. + If a Content-Type header isn't included, one will be added with 'application/json' (except for PUT and forcePost calls). + Query parameters with values of None aren't sent to the server. + Server errors are to be handled by the caller (returned response contains error codes/msgs). + + :param resourcePath: the url to call, not including server address and api version + :param method: REST method, including GET, POST (and forcePost, see below), and PUT (DELETE not yet supported) + :param queryParams: dictionary of query parameters to be added to url, except for forcePost where they are added as 'postData'; not used for PUT calls + :param postData: for POST calls, a dictionary to post; not used for forcePost calls; for PUT calls, name of file to put + :param headerParams: (optional) a dictionary of header data, default None + :param forcePost: (optional) 'force' a POST call using curl (instead of urllib), default False + + :raises RestMethodException: for unrecognized REST method + :raises ServerResponseException: for errors in parsing json response from server, and for urlerrors from the opening url + :returns: Server response deserialized to a python object (dict) + ''' + url = self.apiServerAndVersion + resourcePath + headers = {} + if self.userAgent: + headers['User-Agent'] = self.userAgent + if headerParams: + for param, value in headerParams.iteritems(): + # for param, value in six.iteritems(headerParams): + headers[param] = value + # specify the content type + if not headers.has_key('Content-Type') and not method=='PUT' and not forcePost: + # if not 'Content-Type' in headers and not method=='PUT' and not forcePost: + headers['Content-Type'] = 'application/json' + # include access token in header + headers['Authorization'] = 'Bearer ' + self.apiKey + + data = None + if method == 'GET': + if queryParams: + # Need to remove None values, these should not be sent + sentQueryParams = {} + for param, value in queryParams.iteritems(): + if value != None: + sentQueryParams[param] = value + url = url + '?' + urllib.urlencode(sentQueryParams) + # url = url + '?' + six_urllib.parse.urlencode(sentQueryParams) + request = urllib2.Request(url=url, headers=headers) + # request = six_urllib.request.Request(url=url, headers=headers) + elif method in ['POST', 'PUT', 'DELETE']: + if queryParams: + # Need to remove None values, these should not be sent + sentQueryParams = {} + for param, value in queryParams.iteritems(): + if value != None: + sentQueryParams[param] = value + forcePostUrl = url + url = url + '?' + urllib.urlencode(sentQueryParams) + # url = url + '?' + six_urllib.parse.urlencode(sentQueryParams) + data = postData + if data: + if type(postData) not in [str, int, float, bool]: + data = json.dumps(postData) + if not forcePost: + if data and not len(data): + data='\n' # temp fix, in case is no data in the file, to prevent post request from failing + # THIS IS IT!!!!! : + if isinstance(data, unicode): + # logging.info("dante data class is %s" % data.__class__) + data = data.encode('utf-8') + # pass + + request = urllib2.Request(url=url, headers=headers, data=data)#,timeout=self.timeout) + else: + response = self.__forcePostCall__(forcePostUrl, sentQueryParams, headers) + if method in ['PUT', 'DELETE']: + if method == 'DELETE': + raise NotImplementedError('DELETE REST API calls aren\'t currently supported') + response = self.__putCall__(url, headers, data) + response = response.split()[-1] # discard upload status msg (from curl put?) + else: + raise RestMethodException('Method ' + method + ' is not recognized.') + + # Make the request + if not forcePost and not method in ['PUT', 'DELETE']: # the normal case + try: + response = urllib2.urlopen(request, timeout=self.timeout).read() + except urllib2.HTTPError as e: + response = e.read() # treat http error as a response (handle in caller) + except urllib2.URLError as e: + raise ServerResponseException('URLError: ' + str(e)) + try: + data = json.loads(response) + except ValueError as e: + raise ServerResponseException('Error decoding json in server response') + return data + + def deserialize(self, obj, objClass): + """ + Deserialize a JSON string into a BaseSpacePy object. + + :param obj: A dictionary (or object?) to be deserialized into a class (objClass); or a value to be passed into a new native python type (objClass) + :param objClass: A class object or native python type for the deserialized object, or a string of a class name or native python type. (eg, Project.Project, int, 'Project', 'int') + :returns: A deserialized object + """ + # Create an object class from objClass, if a string was passed in + # Avoid native python types 'file' + if type(objClass) == str: + try: + if (not str(objClass)=='File'): + objClass = eval(objClass.lower()) + else: + objClass = eval(objClass + '.' + objClass) + except NameError: # not a native type, must be model class + objClass = eval(objClass + '.' + objClass) + + # Create an instance of the object class + # If the instance is a native python type, return it + if objClass in [str, int, float, bool]: + return objClass(obj) + instance = objClass() + + # For every swaggerType in the instance that is also in the passed-in obj, + # set the instance value for native python types, + # or recursively deserialize class instances. + # For dynamic types, substitute real class after looking up 'Type' value. + # For lists, deserialize all members of a list, including lists of lists (though not list of list of list...). + # For datetimes, convert to a readable output string + for attr, attrType in instance.swaggerTypes.iteritems(): + if attr in obj: + value = obj[attr] + if attrType in ['str', 'int', 'float', 'bool']: + attrType = eval(attrType) + try: + value = attrType(value) + except UnicodeEncodeError: + value = unicode(value) + setattr(instance, attr, value) + elif attrType == 'DynamicType': + try: + model_name = instance._dynamicType[value['Type']] + except KeyError: + pass + # suppress this warning, which is caused by a bug in BaseSpace + #warn("Warning - unrecognized dynamic type: " + value['Type']) + else: + setattr(instance, attr, self.deserialize(value, model_name)) + elif 'list<' in attrType: + match = re.match('list<(.*)>', attrType) + subClass = match.group(1) + subValues = [] + + # lists of dynamic type + if subClass == 'DynamicType': + for subValue in value: + try: + new_type = instance._dynamicType[subValue['Type']] + except KeyError: + pass + # suppress this warning, which is caused by a bug in BaseSpace + #warn("Warning - unrecognized (list of) dynamic types: " + subValue['Type']) + else: + subValues.append(self.deserialize(subValue, new_type)) + setattr(instance, attr, subValues) + # typical lists + else: + for subValue in value: + subValues.append(self.deserialize(subValue, subClass)) + setattr(instance, attr, subValues) + # list of lists (e.g. map[] property type) + elif 'listoflists<' in attrType: + match = re.match('listoflists<(.*)>', attrType) + subClass = match.group(1) + outvals = [] + for outval in value: + invals = [] + for inval in outval: + invals.append(self.deserialize(inval, subClass)) + outvals.append(invals) + setattr(instance, attr, outvals) + + elif attrType=='dict': + setattr(instance, attr, value) + elif attrType=='datetime': + dt = dateutil.parser.parse(value) + setattr(instance, attr, dt) + else: + # recursive call with attribute type + setattr(instance, attr, self.deserialize(value, attrType)) + return instance diff --git a/src/BaseSpacePy/api/APIClient.py b/src/BaseSpacePy/api/APIClient.py index 94be7d3..7fd84f5 100644 --- a/src/BaseSpacePy/api/APIClient.py +++ b/src/BaseSpacePy/api/APIClient.py @@ -137,7 +137,6 @@ def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None if not forcePost: if data and not len(data): data='\n' # temp fix, in case is no data in the file, to prevent post request from failing - # if isinstance(data, unicode): if data and six.PY3: data = data.encode('utf-8') request = urllib.request.Request(url=url, headers=headers, data=data)#,timeout=self.timeout) @@ -156,7 +155,7 @@ def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None try: response = urllib.request.urlopen(request, timeout=self.timeout).read().decode('utf-8') except urllib.error.HTTPError as e: - response = e.read() # treat http error as a response (handle in caller) + response = e.read().decode('utf-8') # treat http error as a response (handle in caller) except urllib.error.URLError as e: raise ServerResponseException('URLError: ' + str(e)) try: From 07d985050251719b487d7666acf474d74ee66a23 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Fri, 21 Oct 2016 11:33:59 -0700 Subject: [PATCH 14/22] fix unit tests in TestAPIClientMethods --- test/unit_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/unit_tests.py b/test/unit_tests.py index 2e46664..0bfd058 100644 --- a/test/unit_tests.py +++ b/test/unit_tests.py @@ -2140,9 +2140,9 @@ def test__putCall__(self): fileName = os.path.basename(tconst['file_small_upload']), directory = testDir, contentType = tconst['file_small_upload_content_type']) - with open(tconst['file_small_upload']) as fp: + with open(tconst['file_small_upload'], "rb") as fp: out = fp.read() - md5 = hashlib.md5(out).digest().encode('base64') + md5 = base64.b64encode(hashlib.md5(out).digest()) method = 'PUT' resourcePath = '/files/{Id}/parts/{partNumber}' @@ -2236,9 +2236,9 @@ def testCallAPI_PUT(self): fileName = os.path.basename(tconst['file_small_upload']), directory = testDir, contentType = tconst['file_small_upload_content_type']) - with open(tconst['file_small_upload']) as fp: + with open(tconst['file_small_upload'], "rb") as fp: out = fp.read() - md5 = hashlib.md5(out).digest().encode('base64') + md5 = base64.b64encode(hashlib.md5(out).digest()) method = 'PUT' resourcePath = '/files/{Id}/parts/{partNumber}' From cb428b360ccbd79dd151d14189de760cedf6efc9 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Fri, 21 Oct 2016 13:49:58 -0700 Subject: [PATCH 15/22] removed imports which caused a namespcae conflict, fixes TestRunMethods tests --- src/BaseSpacePy/api/APIClient.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/BaseSpacePy/api/APIClient.py b/src/BaseSpacePy/api/APIClient.py index 7fd84f5..1f82db6 100644 --- a/src/BaseSpacePy/api/APIClient.py +++ b/src/BaseSpacePy/api/APIClient.py @@ -4,8 +4,6 @@ import re import io import json -from subprocess import * -import subprocess import dateutil.parser from warnings import warn from BaseSpacePy.model import * From d9f71ae171ba902bf1e845b909b7635719e76552 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Fri, 21 Oct 2016 14:24:56 -0700 Subject: [PATCH 16/22] test fix in progress, logging code not cleaned up yet --- src/BaseSpacePy/api/BaseSpaceAPI.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index 561c053..5a2c996 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -1309,8 +1309,11 @@ def __downloadFile__(self, Id, localDir, name, byteRange=None, standaloneRangeFi :raises DownloadFailedException: if downloaded file size doesn't match the size in BaseSpace :returns: None ''' + logging.debug("in __downloadFile__") if byteRange is None: byteRange = [] + if byteRange is not None: + logging.debug("byteRange is not none!") resourcePath = '/files/{Id}/content' resourcePath = resourcePath.replace('{format}', 'json') method = 'GET' @@ -1351,6 +1354,8 @@ def __downloadFile__(self, Id, localDir, name, byteRange=None, standaloneRangeFi # check that actual downloaded byte size is correct if len(byteRange): expSize = byteRange[1] - byteRange[0] + 1 + logging.debug("len is %d, first is %d, last is %d", len(byteRange), byteRange[0], byteRange[1]) + logging.debug("totRead: %s, expSize: %s", totRead, expSize) if totRead != expSize: raise DownloadFailedException("Ranged download size is not as expected: %d vs %d" % (totRead, expSize)) else: From 256cb9a0b6ad3846d65b6acf1f1fe538c78703a1 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Sun, 23 Oct 2016 12:55:09 -0700 Subject: [PATCH 17/22] unit tests pass locally on py 2 and 3 --- src/BaseSpacePy/api/APIClient.py | 5 +++-- src/BaseSpacePy/api/BaseSpaceAPI.py | 5 ----- src/BaseSpacePy/model/MultipartFileTransfer.py | 14 +++++++------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/BaseSpacePy/api/APIClient.py b/src/BaseSpacePy/api/APIClient.py index 1f82db6..5758dff 100644 --- a/src/BaseSpacePy/api/APIClient.py +++ b/src/BaseSpacePy/api/APIClient.py @@ -130,13 +130,14 @@ def callAPI(self, resourcePath, method, queryParams, postData, headerParams=None url = url + '?' + urllib.parse.urlencode(sentQueryParams) data = postData if data: - if type(postData) not in [str, int, float, bool]: + if type(postData) not in [str, int, float, bool, bytes]: data = json.dumps(postData) if not forcePost: if data and not len(data): data='\n' # temp fix, in case is no data in the file, to prevent post request from failing if data and six.PY3: - data = data.encode('utf-8') + if type(data) is str: + data = data.encode() request = urllib.request.Request(url=url, headers=headers, data=data)#,timeout=self.timeout) else: response = self.__forcePostCall__(forcePostUrl, sentQueryParams, headers) diff --git a/src/BaseSpacePy/api/BaseSpaceAPI.py b/src/BaseSpacePy/api/BaseSpaceAPI.py index 5a2c996..561c053 100755 --- a/src/BaseSpacePy/api/BaseSpaceAPI.py +++ b/src/BaseSpacePy/api/BaseSpaceAPI.py @@ -1309,11 +1309,8 @@ def __downloadFile__(self, Id, localDir, name, byteRange=None, standaloneRangeFi :raises DownloadFailedException: if downloaded file size doesn't match the size in BaseSpace :returns: None ''' - logging.debug("in __downloadFile__") if byteRange is None: byteRange = [] - if byteRange is not None: - logging.debug("byteRange is not none!") resourcePath = '/files/{Id}/content' resourcePath = resourcePath.replace('{format}', 'json') method = 'GET' @@ -1354,8 +1351,6 @@ def __downloadFile__(self, Id, localDir, name, byteRange=None, standaloneRangeFi # check that actual downloaded byte size is correct if len(byteRange): expSize = byteRange[1] - byteRange[0] + 1 - logging.debug("len is %d, first is %d, last is %d", len(byteRange), byteRange[0], byteRange[1]) - logging.debug("totRead: %s, expSize: %s", totRead, expSize) if totRead != expSize: raise DownloadFailedException("Ranged download size is not as expected: %d vs %d" % (totRead, expSize)) else: diff --git a/src/BaseSpacePy/model/MultipartFileTransfer.py b/src/BaseSpacePy/model/MultipartFileTransfer.py index 9058f5f..8a4565c 100644 --- a/src/BaseSpacePy/model/MultipartFileTransfer.py +++ b/src/BaseSpacePy/model/MultipartFileTransfer.py @@ -4,10 +4,12 @@ import math import multiprocessing import shutil +import io import signal import hashlib from subprocess import call import logging +import base64 from BaseSpacePy.api.BaseSpaceException import MultiProcessingTaskFailedException from six.moves import queue @@ -42,10 +44,10 @@ def execute(self, lock): try: fname = os.path.basename(self.local_path) chunk_data = "" - with open(self.local_path) as fh: + with io.open(self.local_path, "rb") as fh: fh.seek(self.piece * self.chunk_size) chunk_data = fh.read(self.chunk_size) - self.md5 = hashlib.md5(chunk_data).digest().encode('base64') + self.md5 = base64.b64encode(hashlib.md5(chunk_data).digest()) try: res = self.api.__uploadMultipartUnit__(self.bs_file_id,self.piece+1,self.md5,chunk_data) except Exception as e: @@ -53,7 +55,7 @@ def execute(self, lock): self.err_msg = str(e) else: # ETag contains hex encoded MD5 of part data on success - if res and 'ETag' in res['Response']: + if res and 'ETag' in res['Response']: self.success = True else: self.success = False @@ -307,11 +309,9 @@ def _setup(self): ''' Determine number of file pieces to upload, add upload tasks to work queue ''' - from math import ceil total_size = os.path.getsize(self.local_path) # round up to get a number of chunks that will be enough for the whole file - fileCount = int(ceil(total_size/float(self.part_size*1024*1024))) - + fileCount = int(math.ceil(total_size/float(self.part_size*1024*1024))) chunk_size = self.part_size*1024*1024 assert chunk_size * fileCount > total_size @@ -401,7 +401,7 @@ def _setup(self): self.file_name = self.bs_file.Name total_bytes = self.bs_file.Size part_size_bytes = self.part_size * (1024**2) - self.file_count = int(math.ceil(total_bytes/part_size_bytes)) + 1 + self.file_count = int(math.ceil(total_bytes/float(part_size_bytes))) file_name = self.file_name if not self.temp_dir: From 254e6f737a013f862c471254da4873a25a883fa8 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Sun, 23 Oct 2016 15:37:10 -0700 Subject: [PATCH 18/22] experimental - add pyflakes checking (but do not fail CI if there are warnings) --- examples/0_Browsing.py | 1 - examples/1_AccessingFiles.py | 1 - examples/2_AppTriggering.py | 2 -- examples/3_Authentication.py | 5 +---- examples/4_AppResultUpload.py | 1 - examples/5_Purchasing.py | 2 -- runtests.sh | 4 ++++ 7 files changed, 5 insertions(+), 11 deletions(-) diff --git a/examples/0_Browsing.py b/examples/0_Browsing.py index 9161ede..faece72 100644 --- a/examples/0_Browsing.py +++ b/examples/0_Browsing.py @@ -14,7 +14,6 @@ """ from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI -import os """ This script demonstrates basic browsing of BaseSpace objects once an access-token diff --git a/examples/1_AccessingFiles.py b/examples/1_AccessingFiles.py index 4d52a0d..e7388aa 100644 --- a/examples/1_AccessingFiles.py +++ b/examples/1_AccessingFiles.py @@ -14,7 +14,6 @@ """ from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI -import os """ This script demonstrates how to access Samples and AppResults from a projects and how to work with the available file data for such instances. diff --git a/examples/2_AppTriggering.py b/examples/2_AppTriggering.py index 8ec00da..d9d7d96 100644 --- a/examples/2_AppTriggering.py +++ b/examples/2_AppTriggering.py @@ -14,8 +14,6 @@ """ from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI -import webbrowser -import time """ This script demonstrates how to retrieve the AppSession object produced diff --git a/examples/3_Authentication.py b/examples/3_Authentication.py index ef15e99..fa72b0a 100644 --- a/examples/3_Authentication.py +++ b/examples/3_Authentication.py @@ -12,11 +12,8 @@ See the License for the specific language governing permissions and limitations under the License. """ -import six -if six.PY2: - from __future__ import print_function +from __future__ import print_function -import sys from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI import time import webbrowser diff --git a/examples/4_AppResultUpload.py b/examples/4_AppResultUpload.py index b8119f1..e36d7ef 100644 --- a/examples/4_AppResultUpload.py +++ b/examples/4_AppResultUpload.py @@ -14,7 +14,6 @@ """ from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI -import os """ This script demonstrates how to create a new AppResults object, change its state diff --git a/examples/5_Purchasing.py b/examples/5_Purchasing.py index 96a67f3..53a224f 100644 --- a/examples/5_Purchasing.py +++ b/examples/5_Purchasing.py @@ -12,10 +12,8 @@ See the License for the specific language governing permissions and limitations under the License. """ -import os import webbrowser import time -from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI from BaseSpacePy.api.BillingAPI import BillingAPI from BaseSpacePy.model.QueryParametersPurchasedProduct import QueryParametersPurchasedProduct as qpp """ diff --git a/runtests.sh b/runtests.sh index 54d70f9..76aa955 100755 --- a/runtests.sh +++ b/runtests.sh @@ -15,6 +15,10 @@ cd .. cat test/dotbasespace/unit_tests.cfg | sed "s/__ACCESS_TOKEN__/$ACCESS_TOKEN/" > ~/.basespace/unit_tests.cfg cp ~/.basespace/unit_tests.cfg ~/.basespace/default.cfg +pip install pyflakes + +find . \( -path ./doc -o -path ./src/build \) -prune -o -name '*.py' -print | xargs pyflakes || true + python test/unit_tests.py From 0da2e44ad9f52d9227a9e62b6faf03eafd99e972 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Sun, 23 Oct 2016 15:44:19 -0700 Subject: [PATCH 19/22] clean up output --- runtests.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/runtests.sh b/runtests.sh index 76aa955..791728b 100755 --- a/runtests.sh +++ b/runtests.sh @@ -17,8 +17,20 @@ cp ~/.basespace/unit_tests.cfg ~/.basespace/default.cfg pip install pyflakes +echo +echo "Static analysis warnings from pyflakes:" +echo +# exclude doc directory because those files are auto-generated find . \( -path ./doc -o -path ./src/build \) -prune -o -name '*.py' -print | xargs pyflakes || true +# TODO add stricter flake8 checking here +# (checks for proper formatting of code in compliance with PEP8) + + +echo +echo "Unit test output:" +echo + python test/unit_tests.py From 27fd4cc61d077b5159bc37b0b87ad56efe90d077 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Sun, 23 Oct 2016 15:47:26 -0700 Subject: [PATCH 20/22] add a comment --- runtests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtests.sh b/runtests.sh index 791728b..2993349 100755 --- a/runtests.sh +++ b/runtests.sh @@ -21,6 +21,8 @@ echo echo "Static analysis warnings from pyflakes:" echo # exclude doc directory because those files are auto-generated +# the "|| true" at the end causes CI not to fail. In future +# you could remove that if you want pyflakes warnings to break the build. find . \( -path ./doc -o -path ./src/build \) -prune -o -name '*.py' -print | xargs pyflakes || true # TODO add stricter flake8 checking here From 2425fd3d6eeebfb8106ff4634af2776689c8d32e Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Thu, 27 Oct 2016 09:41:03 -0700 Subject: [PATCH 21/22] make build report link clickable --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 7d47e84..1bdec89 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -![Build Status](https://api.travis-ci.org/dtenenba/basespace-python-sdk.svg?branch=develop) - +[![Build Status][https://img.shields.io/travis/dtenenba/basespace-python-sdk.svg?style=flat&branch=develop]](https://travis-ci.org/dtenenba/basespace-python-sdk) INTRODUCTION ========================================= From 8d2e90627974f8223ea6e30a075d4645ca5f6e29 Mon Sep 17 00:00:00 2001 From: Dan Tenenbaum Date: Thu, 27 Oct 2016 09:44:09 -0700 Subject: [PATCH 22/22] fix build image --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1bdec89..07f1b7f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -[![Build Status][https://img.shields.io/travis/dtenenba/basespace-python-sdk.svg?style=flat&branch=develop]](https://travis-ci.org/dtenenba/basespace-python-sdk) +[![Build Status][travis-image]](https://travis-ci.org/dtenenba/basespace-python-sdk) + +[travis-image]: https://img.shields.io/travis/dtenenba/basespace-python-sdk.svg?style=flat&branch=develop INTRODUCTION =========================================