diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..fd5df4b --- /dev/null +++ b/.coveragerc @@ -0,0 +1,5 @@ +[report] +exclude_lines = + pragma: no cover + if TYPE_CHECKING.*: + \.\.\. diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index b650b80..6d6a414 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,24 +1,21 @@ ---- +______________________________________________________________________ -name: Bug Report -about: Create a bug report to help us improve pytask-r -title: "BUG:" +name: Bug Report about: Create a bug report to help us improve pytask-r title: "BUG:" labels: "bug" ---- +______________________________________________________________________ - [ ] I have checked that this issue has not already been reported. - [ ] I have confirmed this bug exists on the latest version of pytask-r. -- [ ] (optional) I have confirmed this bug exists on the `main` branch of - pytask-r. +- [ ] (optional) I have confirmed this bug exists on the `main` branch of pytask-r. ---- +______________________________________________________________________ -**Note**: Please read [this -guide](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) detailing -how to provide the necessary information for us to reproduce your bug. +**Note**: Please read +[this guide](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) +detailing how to provide the necessary information for us to reproduce your bug. #### Code Sample, a copy-pastable example diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md index c783a51..98099f5 100644 --- a/.github/ISSUE_TEMPLATE/documentation.md +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -1,11 +1,9 @@ ---- +______________________________________________________________________ -name: Documentation Improvement -about: Report wrong or missing documentation -title: "DOC:" -labels: "documentation" +name: Documentation Improvement about: Report wrong or missing documentation title: +"DOC:" labels: "documentation" ---- +______________________________________________________________________ #### Location of the documentation diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md index feb7247..0f1975d 100644 --- a/.github/ISSUE_TEMPLATE/enhancement.md +++ b/.github/ISSUE_TEMPLATE/enhancement.md @@ -1,16 +1,14 @@ ---- +______________________________________________________________________ -name: Enhancement -about: Suggest an idea for pytask-r -title: "ENH:" -labels: "enhancement" +name: Enhancement about: Suggest an idea for pytask-r title: "ENH:" labels: +"enhancement" ---- +______________________________________________________________________ #### Is your feature request related to a problem? -Provide a description of what the problem is, e.g. "I wish I could use pytask-r -to do [...]". +Provide a description of what the problem is, e.g. "I wish I could use pytask-r to do +\[...\]". #### Describe the solution you'd like diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index bd8d2e3..da1e72d 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -1,17 +1,15 @@ ---- +______________________________________________________________________ -name: Submit Question -about: Ask a general question about pytask-r -title: "QST:" -labels: "question" +name: Submit Question about: Ask a general question about pytask-r title: "QST:" labels: +"question" ---- +______________________________________________________________________ #### Question about pytask-r -**Note**: If you'd still like to submit a question, please read [this guide]( -https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) detailing how to -provide the necessary information for us to reproduce your question. +**Note**: If you'd still like to submit a question, please read +[this guide](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) +detailing how to provide the necessary information for us to reproduce your question. ```python # Your code here, if applicable diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1817a2f..b627f37 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -6,4 +6,4 @@ Provide a description and/or bullet points to describe the changes in this PR. - [ ] Reference issues which can be closed due to this PR with "Closes #x". - [ ] Review whether the documentation needs to be updated. -- [ ] Document PR in docs/changes.rst. +- [ ] Document PR in CHANGES.md. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6e6bff1..def8cca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: run-tests: - name: Run tests for ${{ matrix.os }} on ${{ matrix.python-version }} and ${{ matrix.r-version }} + name: Run tests for ${{ matrix.os }} on ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: @@ -25,7 +25,6 @@ jobs: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.7', '3.8', '3.9', '3.10'] - r-version: ['3.6.3'] steps: - uses: actions/checkout@v2 @@ -45,29 +44,15 @@ jobs: run: tox -e pytest -- -m "unit or (not integration and not end_to_end)" --cov=./ --cov-report=xml -n auto - name: Upload coverage report for unit tests and doctests. - if: runner.os == 'Linux' && matrix.python-version == '3.9' && matrix.r-version == '3.6.3' + if: runner.os == 'Linux' && matrix.python-version == '3.9' shell: bash -l {0} run: bash <(curl -s https://codecov.io/bash) -F unit -c - # - name: Run integration tests. - # shell: bash -l {0} - # run: tox -e pytest -- -m integration --cov=./ --cov-report=xml -n auto - - # - name: Upload coverage reports of integration tests. - # if: runner.os == 'Linux' && matrix.python-version == '3.9' && matrix.r-version == '3.6.3' - # shell: bash -l {0} - # run: bash <(curl -s https://codecov.io/bash) -F integration -c - - name: Run end-to-end tests. shell: bash -l {0} run: tox -e pytest -- -m end_to_end --cov=./ --cov-report=xml -n auto - name: Upload coverage reports of end-to-end tests. - if: runner.os == 'Linux' && matrix.python-version == '3.9' && matrix.r-version == '3.6.3' + if: runner.os == 'Linux' && matrix.python-version == '3.9' shell: bash -l {0} run: bash <(curl -s https://codecov.io/bash) -F end_to_end -c - - - name: Validate codecov.yml - if: runner.os == 'Linux' && matrix.python-version == '3.9' && matrix.r-version == '3.6.3' - shell: bash -l {0} - run: cat codecov.yml | curl --data-binary @- https://codecov.io/validate diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f37cffd..4c09c56 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,9 +16,6 @@ repos: - id: python-no-eval - id: python-no-log-warn - id: python-use-type-annotations - - id: rst-backticks - - id: rst-directive-colons - - id: rst-inline-touching-normal - id: text-unicode-replacement-char - repo: https://github.com/asottile/pyupgrade rev: v2.32.0 @@ -38,11 +35,6 @@ repos: rev: 22.3.0 hooks: - id: black -- repo: https://github.com/asottile/blacken-docs - rev: v1.12.1 - hooks: - - id: blacken-docs - additional_dependencies: [black] - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: @@ -64,24 +56,30 @@ repos: pydocstyle, Pygments, ] -- repo: https://github.com/PyCQA/doc8 - rev: 0.11.1 - hooks: - - id: doc8 - repo: https://github.com/econchick/interrogate rev: 1.5.0 hooks: - id: interrogate args: [-v, --fail-under=40, src, tests] +- repo: https://github.com/executablebooks/mdformat + rev: 0.7.14 + hooks: + - id: mdformat + additional_dependencies: [ + mdformat-gfm, + mdformat-black, + ] + args: [--wrap, "88"] - repo: https://github.com/codespell-project/codespell rev: v2.1.0 hooks: - id: codespell - args: [-L unparseable] - repo: https://github.com/mgedmin/check-manifest rev: "0.48" hooks: - id: check-manifest + args: [--no-build-isolation] + additional_dependencies: [setuptools-scm, toml] - repo: meta hooks: - id: check-hooks-apply diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..e44d3e5 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,61 @@ +# Changes + +This is a record of all past pytask-r releases and what went into them in reverse +chronological order. Releases follow [semantic versioning](https://semver.org/) and all +releases are available on [PyPI](https://pypi.org/project/pytask-r) and +[Anaconda.org](https://anaconda.org/conda-forge/pytask-r). + +## 0.2.0 - 2022-xx-xx + +- {pull}`24` removes an unnecessary hook implementation. + +## 0.1.1 - 2022-02-08 + +- {pull}`22` skips concurrent CI builds. +- {pull}`23` deprecate Python 3.6 and add support for Python 3.10. + +## 0.1.0 - 2021-07-22 + +- {pull}`15` fixes the tests related to changes in pytask. +- {pull}`16` replaces versioneer with setuptools-scm. +- {pull}`18` aligns pytask-r with pytask v0.1.0. + +## 0.0.9 - 2021-03-05 + +- {pull}`13` fixes the version number in the release. + +## 0.0.8 - 2021-03-03 + +- {pull}`10` fixes some post-release issues. +- {pull}`12` add dependencies to `setup.py`. + +## 0.0.7 - 2021-02-25 + +- {pull} `9` publishes pytask-r on PyPI, adds versioneer, and more. + +## 0.0.6 - 2021-01-16 + +- {pull} `7` converts command line arguments automatically to string and removes the + `--vanilla` option as the default. + +## 0.0.5 - 2020-10-30 + +- {pull} `5` makes pytask-r work with pytask v0.0.9. + +## 0.0.4 - 2020-10-14 + +- {pull} `4` fixes `pytask_collect_task_teardown` with non-R tasks and releases 0.0.4. + +## 0.0.3 - 2020-10-04 + +- {pull} `2` fixes marker description and adds some changes to the readme. +- {pull} `3` check for exit code equal to 0, preparation for pytask v0.0.7, and releases + v0.0.3. + +## 0.0.2 - 2020-08-14 + +- {pull} `1` includes some minor changes like fixing the missing version variable. + +## 0.0.1 - 2020-07-22 + +- Release v0.0.1. diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index fc937c2..0000000 --- a/CHANGES.rst +++ /dev/null @@ -1,86 +0,0 @@ -Changes -======= - -This is a record of all past pytask-r releases and what went into them in reverse -chronological order. Releases follow `semantic versioning `_ and -all releases are available on `PyPI `_ and -`Anaconda.org `_. - - -0.2.0 - 2022-xx-xx ------------------- - -- :gh:`24` removes an unnecessary hook implementation. - - -0.1.1 - 2022-02-08 ------------------- - -- :gh:`22` skips concurrent CI builds. -- :gh:`23` deprecate Python 3.6 and add support for Python 3.10. - - -0.1.0 - 2021-07-22 ------------------- - -- :gh:`15` fixes the tests related to changes in pytask. -- :gh:`16` replaces versioneer with setuptools-scm. -- :gh:`18` aligns pytask-r with pytask v0.1.0. - - -0.0.9 - 2021-03-05 ------------------- - -- :gh:`13` fixes the version number in the release. - - -0.0.8 - 2021-03-03 ------------------- - -- :gh:`10` fixes some post-release issues. -- :gh:`12` add dependencies to ``setup.py``. - - -0.0.7 - 2021-02-25 ------------------- - -- :gh:`9` publishes pytask-r on PyPI, adds versioneer, and more. - - -0.0.6 - 2021-01-16 ------------------- - -- :gh:`7` converts command line arguments automatically to string and removes the - ``--vanilla`` option as the default. - - -0.0.5 - 2020-10-30 ------------------- - -- :gh:`5` makes pytask-r work with pytask v0.0.9. - - -0.0.4 - 2020-10-14 ------------------- - -- :gh:`4` fixes ``pytask_collect_task_teardown`` with non-R tasks and releases 0.0.4. - - -0.0.3 - 2020-10-04 ------------------- - -- :gh:`2` fixes marker description and adds some changes to the readme. -- :gh:`3` check for exit code equal to 0, preparation for pytask v0.0.7, and releases - v0.0.3. - - -0.0.2 - 2020-08-14 ------------------- - -- :gh:`1` includes some minor changes like fixing the missing version variable. - - -0.0.1 - 2020-07-22 ------------------- - -- Release v0.0.1. diff --git a/LICENSE b/LICENSE index 4c96cf3..f4c44ec 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2020-2021 Tobias Raabe +Copyright 2020 Tobias Raabe Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software diff --git a/MANIFEST.in b/MANIFEST.in index eb71604..ebba16e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,12 +1,10 @@ -prune .conda prune tests -exclude *.rst +exclude .coveragerc +exclude *.md exclude *.yml exclude *.yaml exclude tox.ini include LICENSE -include README.rst -include versioneer.py -include src/pytask_r/_version.py +include README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..7d90f84 --- /dev/null +++ b/README.md @@ -0,0 +1,256 @@ +# pytask-r + +[![PyPI](https://img.shields.io/pypi/v/pytask-r?color=blue)](https://pypi.org/project/pytask-r) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pytask-r)](https://pypi.org/project/pytask-r) +[![image](https://img.shields.io/conda/vn/conda-forge/pytask-r.svg)](https://anaconda.org/conda-forge/pytask-r) +[![image](https://img.shields.io/conda/pn/conda-forge/pytask-r.svg)](https://anaconda.org/conda-forge/pytask-r) +[![PyPI - License](https://img.shields.io/pypi/l/pytask-r)](https://pypi.org/project/pytask-r) +[![image](https://img.shields.io/github/workflow/status/pytask-dev/pytask-r/main/main)](https://github.com/pytask-dev/pytask-r/actions?query=branch%3Amain) +[![image](https://codecov.io/gh/pytask-dev/pytask-r/branch/main/graph/badge.svg)](https://codecov.io/gh/pytask-dev/pytask-r) +[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pytask-dev/pytask-r/main.svg)](https://results.pre-commit.ci/latest/github/pytask-dev/pytask-r/main) +[![image](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + +______________________________________________________________________ + +Run R scripts with pytask. + +## Installation + +pytask-r is available on [PyPI](https://pypi.org/project/pytask-r) and +[Anaconda.org](https://anaconda.org/conda-forge/pytask-r). Install it with + +```console +$ pip install pytask-r + +# or + +$ conda install -c conda-forge pytask-r +``` + +You also need to have R installed and `Rscript` on your command line. Test it by typing +the following on the command line + +```console +$ Rscript --help +``` + +If an error is shown instead of a help page, you can install R with `conda` by choosing +either R or Microsoft R Open (MRO). Choose one of the two following commands. (See +[here](https://docs.anaconda.com/anaconda/user-guide/tasks/%20using-r-language) for +further explanation on Anaconda, R, and MRO.) + +```console +$ conda install -c r r-base # For normal R. +$ conda install -c r mro-base # For MRO. +``` + +Or install install R from the official [R Project](https://www.r-project.org/). + +## Usage + +To create a task which runs a R script, define a task function with the `@pytask.mark.r` +decorator. The `script` keyword provides an absolute path or path relative to the task +module to the R script. + +```python +import pytask + + +@pytask.mark.r(script="script.r") +@pytask.mark.produces("out.rds") +def task_run_r_script(): + pass +``` + +If you are wondering why the function body is empty, know that pytask-r replaces the +body with a predefined internal function. See the section on implementation details for +more information. + +### Dependencies and Products + +Dependencies and products can be added as with a normal pytask task using the +`@pytask.mark.depends_on` and `@pytask.mark.produces` decorators. which is explained in +this +[tutorial](https://pytask-dev.readthedocs.io/en/stable/tutorials/defining_dependencies_products.html). + +### Accessing dependencies and products in the script + +To access the paths of dependencies and products in the script, pytask-r stores the +information by default in a `.json` file. The path to this file is passed as a +positional argument to the script. Inside the script, you can read the information. + +```r +library(jsonlite) + +args <- commandArgs(trailingOnly=TRUE) + +path_to_json <- args[length(args)] + +config <- read_json(path_to_json) + +config$produces # Is the path to the output file "../out.csv". +``` + +The `.json` file is stored in the same folder as the task in a `.pytask` directory. + +To parse the JSON file, you need to install +[jsonlite](https://github.com/jeroen/jsonlite). + +You can also pass any other information to your script by using the `@pytask.mark.task` +decorator. + +```python +@pytask.mark.task(kwargs={"number": 1}) +@pytask.mark.r(script="script.r") +@pytask.mark.produces("out.rds") +def task_run_r_script(): + pass +``` + +and inside the script use + +```r +config$number # Is 1. +``` + +### Debugging + +In case a task throws an error, you might want to execute the script independently from +pytask. After a failed execution, you see the command which executed the R script in the +report of the task. It looks roughly like this + +```console +$ Rscript script.r /.pytask/task_py_task_example.json +``` + +### Command Line Arguments + +The decorator can be used to pass command line arguments to `Rscript`. See the following +example. + +```python +@pytask.mark.r(script="script.r", options="--vanilla") +@pytask.mark.produces("out.rds") +def task_run_r_script(): + pass +``` + +### Repeating tasks with different scripts or inputs + +You can also repeat the execution of tasks, meaning executing multiple R scripts or +passing different command line arguments to the same R script. + +The following task executes two R scripts, `script_1.r` and `script_2.r`, which produce +different outputs. + +```python +for i in range(2): + + @pytask.mark.task + @pytask.mark.r(script=f"script_{i}.r") + @pytask.mark.produces(f"out_{i}.csv") + def task_execute_r_script(): + pass +``` + +If you want to pass different inputs to the same R script, pass these arguments with the +`kwargs` keyword of the `@pytask.mark.task` decorator. + +```python +for i in range(2): + + @pytask.mark.task(kwargs={"i": i}) + @pytask.mark.r(script="script.r") + @pytask.mark.produces(f"output_{i}.csv") + def task_execute_r_script(): + pass +``` + +and inside the task access the argument `i` with + +```r +library(jsonlite) + +args <- commandArgs(trailingOnly=TRUE) + +path_to_json <- args[length(args)] + +config <- read_json(path_to_json) + +config$produces # Is the path to the output file "../output_{i}.csv". + +config$i # Is the number. +``` + +### Serializers + +You can also serialize your data with any other tool you like. By default, pytask-r also +supports YAML (if PyYaml is installed). + +Use the `serializer` keyword arguments of the `@pytask.mark.r` decorator with + +```python +@pytask.mark.r(script="script.r", serializer="yaml") +def task_example(): + ... +``` + +And in your R script use + +```r +library(yaml) +args <- commandArgs(trailingOnly=TRUE) +config <- read_yaml(args[length(args)]) +``` + +Note that the `YAML` package needs to be installed. + +If you need a custom serializer, you can also provide any callable to `serializer` which +transforms data to a string. Use `suffix` to set the correct file ending. + +Here is a replication of the JSON example. + +```python +import json + + +@pytask.mark.r(script="script.r", serializer=json.dumps, suffix=".json") +def task_example(): + ... +``` + +### Configuration + +You can influence the default behavior of pytask-r with some configuration values. + +**`r_serializer`** + +Use this option to change the default serializer. + +```toml +[tool.pytask.ini_options] +r_serializer = "json" +``` + +**`r_suffix`** + +Use this option to set the default suffix of the file which contains serialized paths to +dependencies and products and more. + +```toml +[tool.pytask.ini_options] +r_suffix = ".json" +``` + +**`r_options`** + +Use this option to set default options for each task which are separated by whitespace. + +```toml +[tool.pytask.ini_options] +r_options = ["--vanilla"] +``` + +## Changes + +Consult the [release notes](CHANGES.md) to find out about what is new. diff --git a/README.rst b/README.rst deleted file mode 100644 index 7719994..0000000 --- a/README.rst +++ /dev/null @@ -1,244 +0,0 @@ -.. image:: https://img.shields.io/pypi/v/pytask-r?color=blue - :alt: PyPI - :target: https://pypi.org/project/pytask-r - -.. image:: https://img.shields.io/pypi/pyversions/pytask-r - :alt: PyPI - Python Version - :target: https://pypi.org/project/pytask-r - -.. image:: https://img.shields.io/conda/vn/conda-forge/pytask-r.svg - :target: https://anaconda.org/conda-forge/pytask-r - -.. image:: https://img.shields.io/conda/pn/conda-forge/pytask-r.svg - :target: https://anaconda.org/conda-forge/pytask-r - -.. image:: https://img.shields.io/pypi/l/pytask-r - :alt: PyPI - License - :target: https://pypi.org/project/pytask-r - -.. image:: https://img.shields.io/github/workflow/status/pytask-dev/pytask-r/main/main - :target: https://github.com/pytask-dev/pytask-r/actions?query=branch%3Amain - -.. image:: https://codecov.io/gh/pytask-dev/pytask-r/branch/main/graph/badge.svg - :target: https://codecov.io/gh/pytask-dev/pytask-r - -.. image:: https://results.pre-commit.ci/badge/github/pytask-dev/pytask-r/main.svg - :target: https://results.pre-commit.ci/latest/github/pytask-dev/pytask-r/main - :alt: pre-commit.ci status - -.. image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black - ------- - -pytask-r -======== - -Run R scripts with pytask. - - -Installation ------------- - -pytask-r is available on `PyPI `_ and `Anaconda.org -`_. Install it with - -.. code-block:: console - - $ pip install pytask-r - - # or - - $ conda install -c conda-forge pytask-r - -You also need to have R installed and ``Rscript`` on your command line. Test it by -typing the following on the command line - -.. code-block:: console - - $ Rscript --help - -If an error is shown instead of a help page, you can install R with ``conda`` by -choosing either R or Microsoft R Open (MRO). Choose one of the two following commands. -(See `here `_ -for further explanation on Anaconda, R, and MRO.) - -.. code-block:: console - - $ conda install -c r r-base # For normal R. - $ conda install -c r mro-base # For MRO. - -Or install install R from the official `R Project `_. - - -Usage ------ - -Similarly to normal task functions which execute Python code, you define tasks to -execute scripts written in R with Python functions. The difference is that the function -body does not contain any logic, but the decorator tells pytask how to handle the task. - -Here is an example where you want to run ``script.r``. - -.. code-block:: python - - import pytask - - - @pytask.mark.r - @pytask.mark.depends_on("script.r") - @pytask.mark.produces("out.rds") - def task_run_r_script(): - pass - -Note that, you need to apply the ``@pytask.mark.r`` marker so that pytask-r handles the -task. - -If you are wondering why the function body is empty, know that pytask-r replaces the -body with a predefined internal function. See the section on implementation details for -more information. - - -Multiple dependencies and products -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -What happens if a task has more dependencies? Using a list, the R script which should be -executed must be found in the first position of the list. - -.. code-block:: python - - @pytask.mark.r - @pytask.mark.depends_on(["script.r", "input.rds"]) - @pytask.mark.produces("out.rds") - def task_run_r_script(): - pass - -If you use a dictionary to pass dependencies to the task, pytask-r will, first, look -for a ``"source"`` key in the dictionary and, secondly, under the key ``0``. - -.. code-block:: python - - @pytask.mark.r - @pytask.mark.depends_on({"source": "script.r", "input": "input.rds"}) - def task_run_r_script(): - pass - - - # or - - - @pytask.mark.r - @pytask.mark.depends_on({0: "script.r", "input": "input.rds"}) - def task_run_r_script(): - pass - - - # or two decorators for the function, if you do not assign a name to the input. - - - @pytask.mark.r - @pytask.mark.depends_on({"source": "script.r"}) - @pytask.mark.depends_on("input.rds") - def task_run_r_script(): - pass - - -Command Line Arguments -~~~~~~~~~~~~~~~~~~~~~~ - -The decorator can be used to pass command line arguments to ``Rscript``. See the -following example. - -.. code-block:: python - - @pytask.mark.r("value") - @pytask.mark.depends_on("script.r") - @pytask.mark.produces("out.rds") - def task_run_r_script(): - pass - -And in your ``script.r``, you can intercept the value with - -.. code-block:: r - - args <- commandArgs(trailingOnly=TRUE) - arg <- args[1] # holds ``"value"`` - - -Parametrization -~~~~~~~~~~~~~~~ - -You can also parametrize the execution of scripts, meaning executing multiple R scripts -as well as passing different command line arguments to the same R script. - -The following task executes two R scripts which produce different outputs. - -.. code-block:: python - - from src.config import BLD, SRC - - - @pytask.mark.r - @pytask.mark.parametrize( - "depends_on, produces", - [(SRC / "script_1.r", BLD / "1.rds"), (SRC / "script_2.r", BLD / "2.rds")], - ) - def task_execute_r_script(): - pass - -And the R script includes something like - -.. code-block:: r - - args <- commandArgs(trailingOnly=TRUE) - produces <- args[1] # holds the path - -If you want to pass different command line arguments to the same R script, you have to -include the ``@pytask.mark.r`` decorator in the parametrization just like with -``@pytask.mark.depends_on`` and ``@pytask.mark.produces``. - -.. code-block:: python - - @pytask.mark.depends_on("script.r") - @pytask.mark.parametrize( - "produces, r", - [(BLD / "output_1.rds", "1"), (BLD / "output_2.rds", "2")], - ) - def task_execute_r_script(): - pass - - -Configuration -------------- - -If you want to change the name of the key which identifies the R script, change the -following default configuration in your pytask configuration file. - -.. code-block:: ini - - r_source_key = source - - -Implementation Details ----------------------- - -The plugin is a convenient wrapper around - -.. code-block:: python - - import subprocess - - subprocess.run(["Rscript", "script.r"], check=True) - -to which you can always resort to when the plugin does not deliver functionality you -need. - -It is not possible to enter a post-mortem debugger when an error happens in the R script -or enter the debugger when starting the script. If there exists a solution for that, -hints as well as contributions are highly appreciated. - - -Changes -------- - -Consult the `release notes `_ to find out about what is new. diff --git a/codecov.yml b/codecov.yml index 94db886..bd9b2df 100644 --- a/codecov.yml +++ b/codecov.yml @@ -21,6 +21,4 @@ coverage: ignore: - ".tox/**/*" - - "setup.py" - - "versioneer.py" - "src/pytask_r/_version.py" diff --git a/environment.yml b/environment.yml index 4940c2d..836ac2c 100644 --- a/environment.yml +++ b/environment.yml @@ -16,15 +16,20 @@ dependencies: - conda-verify # Package dependencies - - pytask >=0.1.0 - - pytask-parallel >=0.1.0 - - r-base + - pytask >=0.2 + - pytask-parallel >=0.2 + + - r-base >4 + - r-jsonlite + - r-yaml # Misc - - bumpversion - jupyterlab - - pdbpp - pre-commit - pytest-cov - pytest-xdist + - tox-conda - black + + - pip: + - -e . diff --git a/pyproject.toml b/pyproject.toml index 17fb0c4..afa5c36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,6 @@ [build-system] requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.0"] +build-backend = "setuptools.build_meta" [tool.setuptools_scm] diff --git a/setup.cfg b/setup.cfg index fe11d45..e0abd28 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,8 @@ [metadata] name = pytask_r description = Run R scripts with pytask. -long_description = file: README.rst -long_description_content_type = text/x-rst +long_description = file: README.md +long_description_content_type = text/markdown url = https://github.com/pytask-dev/pytask-r author = Tobias Raabe author_email = raabe@posteo.de @@ -10,7 +10,7 @@ license = MIT license_file = LICENSE platforms = any classifiers = - Development Status :: 3 - Alpha + Development Status :: 4 - Beta License :: OSI Approved :: MIT License Operating System :: OS Independent Programming Language :: Python :: 3 @@ -21,7 +21,7 @@ classifiers = Programming Language :: Python :: 3.10 Programming Language :: R project_urls = - Changelog = https://github.com/pytask-dev/pytask-r/blob/main/CHANGES.rst + Changelog = https://github.com/pytask-dev/pytask-r/blob/main/CHANGES.md Documentation = https://github.com/pytask-dev/pytask-r Github = https://github.com/pytask-dev/pytask-r Tracker = https://github.com/pytask-dev/pytask-r/issues @@ -30,7 +30,8 @@ project_urls = packages = find: install_requires = click - pytask>=0.1.7 + pybaum>=0.1.1 + pytask>=0.2 python_requires = >=3.7 include_package_data = True package_dir = =src diff --git a/setup.py b/setup.py deleted file mode 100644 index c21a9ee..0000000 --- a/setup.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import annotations - -from setuptools import setup - - -if __name__ == "__main__": - setup() diff --git a/src/pytask_r/collect.py b/src/pytask_r/collect.py index fe0daaa..739a511 100644 --- a/src/pytask_r/collect.py +++ b/src/pytask_r/collect.py @@ -1,107 +1,137 @@ """Collect tasks.""" from __future__ import annotations -import copy import functools import subprocess -from typing import Iterable -from typing import Sequence - -from _pytask.config import hookimpl -from _pytask.mark_utils import get_specific_markers_from_task -from _pytask.nodes import FilePathNode -from _pytask.parametrize import _copy_func - - -def r(options: str | Iterable[str] | None = None): - """Specify command line options for Rscript. - - Parameters - ---------- - options : Optional[Union[str, Iterable[str]]] - One or multiple command line options passed to Rscript. - - """ - options = [] if options is None else _to_list(options) - options = [str(i) for i in options] - return options - - -def run_r_script(r): +from pathlib import Path +from types import FunctionType + +from pytask import depends_on +from pytask import has_mark +from pytask import hookimpl +from pytask import Mark +from pytask import parse_nodes +from pytask import produces +from pytask import remove_marks +from pytask import Task +from pytask_r.serialization import SERIALIZER +from pytask_r.shared import r + + +def run_r_script(script: Path, options: list[str], serialized: Path) -> None: """Run an R script.""" - print("Executing " + " ".join(r) + ".") # noqa: T001 - subprocess.run(r, check=True) + cmd = ["Rscript", script.as_posix(), *options, str(serialized)] + print("Executing " + " ".join(cmd) + ".") # noqa: T001 + subprocess.run(cmd, check=True) @hookimpl -def pytask_collect_task_teardown(session, task): +def pytask_collect_task(session, path, name, obj): """Perform some checks.""" - if get_specific_markers_from_task(task, "r"): - source = _get_node_from_dictionary(task.depends_on, "source") - if isinstance(source, FilePathNode) and source.value.suffix not in [".r", ".R"]: - raise ValueError( - "The first dependency of an R task must be the executable script." - ) - - r_function = _copy_func(run_r_script) - r_function.pytaskmark = copy.deepcopy(task.function.pytaskmark) + __tracebackhide__ = True - merged_marks = _merge_all_markers(task) - args = r(*merged_marks.args, **merged_marks.kwargs) - options = _prepare_cmd_options(session, task, args) - r_function = functools.partial(r_function, r=options) - - task.function = r_function - - -def _get_node_from_dictionary(obj, key, fallback=0): - """Get node from dictionary.""" - if isinstance(obj, dict): - obj = obj.get(key) or obj.get(fallback) - return obj + if ( + (name.startswith("task_") or has_mark(obj, "task")) + and callable(obj) + and has_mark(obj, "r") + ): + obj, marks = remove_marks(obj, "r") + if len(marks) > 1: + raise ValueError( + f"Task {name!r} has multiple @pytask.mark.r marks, but only one is " + "allowed." + ) -def _merge_all_markers(task): - """Combine all information from markers for the compile r function.""" - r_marks = get_specific_markers_from_task(task, "r") - mark = r_marks[0] - for mark_ in r_marks[1:]: - mark = mark.combined_with(mark_) + mark = _parse_r_mark( + mark=marks[0], + default_options=session.config["r_options"], + default_serializer=session.config["r_serializer"], + default_suffix=session.config["r_suffix"], + ) + script, options, _, _ = r(**marks[0].kwargs) + + obj.pytask_meta.markers.append(mark) + + dependencies = parse_nodes(session, path, name, obj, depends_on) + products = parse_nodes(session, path, name, obj, produces) + + markers = obj.pytask_meta.markers if hasattr(obj, "pytask_meta") else [] + kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {} + + task = Task( + base_name=name, + path=path, + function=_copy_func(run_r_script), + depends_on=dependencies, + produces=products, + markers=markers, + kwargs=kwargs, + ) + + script_node = session.hook.pytask_collect_node( + session=session, path=path, node=script + ) + + if isinstance(task.depends_on, dict): + task.depends_on["__script"] = script_node + task.attributes["r_keep_dict"] = True + else: + task.depends_on = {0: task.depends_on, "__script": script_node} + task.attributes["r_keep_dict"] = False + + task.function = functools.partial( + task.function, script=task.depends_on["__script"].path, options=options + ) + + return task + + +def _parse_r_mark(mark, default_options, default_serializer, default_suffix): + """Parse a Julia mark.""" + script, options, serializer, suffix = r(**mark.kwargs) + + parsed_kwargs = {} + for arg_name, value, default in [ + ("script", script, None), + ("options", options, default_options), + ("serializer", serializer, default_serializer), + ]: + parsed_kwargs[arg_name] = value if value else default + + if ( + isinstance(parsed_kwargs["serializer"], str) + and parsed_kwargs["serializer"] in SERIALIZER + ): + proposed_suffix = SERIALIZER[parsed_kwargs["serializer"]]["suffix"] + else: + proposed_suffix = default_suffix + parsed_kwargs["suffix"] = suffix if suffix else proposed_suffix + + mark = Mark("r", (), parsed_kwargs) return mark -def _prepare_cmd_options(session, task, args): - """Prepare the command line arguments to execute the do-file. +def _copy_func(func: FunctionType) -> FunctionType: + """Create a copy of a function. - The last entry changes the name of the log file. We take the task id as a name which - is unique and does not cause any errors when parallelizing the execution. + Based on https://stackoverflow.com/a/13503277/7523785. - """ - source = _get_node_from_dictionary(task.depends_on, session.config["r_source_key"]) - return ["Rscript", source.path.as_posix(), *args] - - -def _to_list(scalar_or_iter): - """Convert scalars and iterables to list. - - Parameters - ---------- - scalar_or_iter : str or list - - Returns + Example ------- - list - - Examples - -------- - >>> _to_list("a") - ['a'] - >>> _to_list(["b"]) - ['b'] + >>> def _func(): pass + >>> copied_func = _copy_func(_func) + >>> _func is copied_func + False """ - return ( - [scalar_or_iter] - if isinstance(scalar_or_iter, str) or not isinstance(scalar_or_iter, Sequence) - else list(scalar_or_iter) + new_func = FunctionType( + func.__code__, + func.__globals__, + name=func.__name__, + argdefs=func.__defaults__, + closure=func.__closure__, ) + new_func = functools.update_wrapper(new_func, func) + new_func.__kwdefaults__ = func.__kwdefaults__ + return new_func diff --git a/src/pytask_r/config.py b/src/pytask_r/config.py index 44e139b..faabb81 100644 --- a/src/pytask_r/config.py +++ b/src/pytask_r/config.py @@ -1,11 +1,27 @@ """Configure pytask.""" from __future__ import annotations -from _pytask.config import hookimpl +from pytask import hookimpl @hookimpl def pytask_parse_config(config, config_from_file): """Register the r marker.""" config["markers"]["r"] = "Tasks which are executed with Rscript." - config["r_source_key"] = config_from_file.get("r_source_key", "source") + config["r_serializer"] = config_from_file.get("r_serializer", "json") + config["r_suffix"] = config_from_file.get("r_suffix", "") + config["r_options"] = _parse_value_or_whitespace_option( + config_from_file.get("r_options") + ) + + +def _parse_value_or_whitespace_option(value: str | None) -> None | str | list[str]: + """Parse option which can hold a single value or values separated by new lines.""" + if value in ["none", "None", None, ""]: + return None + elif isinstance(value, list): + return list(map(str, value)) + elif isinstance(value, str): + return [v.strip() for v in value.split(" ") if v.strip()] + else: + raise ValueError(f"Input {value!r} is neither a 'str' nor 'None'.") diff --git a/src/pytask_r/execute.py b/src/pytask_r/execute.py index 3346804..8d26982 100644 --- a/src/pytask_r/execute.py +++ b/src/pytask_r/execute.py @@ -1,17 +1,58 @@ """Execute tasks.""" from __future__ import annotations +import functools import shutil +from typing import Any -from _pytask.config import hookimpl -from _pytask.mark_utils import get_specific_markers_from_task +from pybaum.tree_util import tree_map +from pytask import get_marks +from pytask import hookimpl +from pytask import Task +from pytask_r.serialization import create_path_to_serialized +from pytask_r.serialization import serialize_keyword_arguments +from pytask_r.shared import r @hookimpl def pytask_execute_task_setup(task): """Perform some checks when a task marked with the r marker is executed.""" - if get_specific_markers_from_task(task, "r"): + marks = get_marks(task, "r") + if marks: if shutil.which("Rscript") is None: raise RuntimeError( "Rscript is needed to run R scripts, but it is not found on your PATH." ) + + assert len(marks) == 1 + + _, _, serializer, suffix = r(**marks[0].kwargs) + + path_to_serialized = create_path_to_serialized(task, suffix) + path_to_serialized.parent.mkdir(parents=True, exist_ok=True) + task.function = functools.partial( + task.function, + serialized=path_to_serialized, + ) + kwargs = collect_keyword_arguments(task) + serialize_keyword_arguments(serializer, path_to_serialized, kwargs) + + +def collect_keyword_arguments(task: Task) -> dict[str, Any]: + """Collect keyword arguments for function.""" + # Remove all kwargs from the task so that they are not passed to the function. + kwargs = dict(task.kwargs) + task.kwargs = {} + + if len(task.depends_on) == 1 and "__script" in task.depends_on: + pass + elif not task.attributes["r_keep_dict"] and len(task.depends_on) == 2: + kwargs["depends_on"] = str(task.depends_on[0].value) + else: + kwargs["depends_on"] = tree_map(lambda x: str(x.value), task.depends_on) + kwargs["depends_on"].pop("__script") + + if task.produces: + kwargs["produces"] = tree_map(lambda x: str(x.value), task.produces) + + return kwargs diff --git a/src/pytask_r/parametrize.py b/src/pytask_r/parametrize.py index 6c3c278..0f5cefc 100644 --- a/src/pytask_r/parametrize.py +++ b/src/pytask_r/parametrize.py @@ -1,8 +1,8 @@ """Parametrize tasks.""" from __future__ import annotations -from _pytask.config import hookimpl -from _pytask.mark import MARK_GEN as mark # noqa: N811 +import pytask +from pytask import hookimpl @hookimpl @@ -10,4 +10,4 @@ def pytask_parametrize_kwarg_to_marker(obj, kwargs): """Attach parametrized r arguments to the function with a marker.""" if callable(obj): if "r" in kwargs: - mark.r(kwargs.pop("r"))(obj) + pytask.mark.r(**kwargs.pop("r"))(obj) diff --git a/src/pytask_r/plugin.py b/src/pytask_r/plugin.py index 7677758..5440ed8 100644 --- a/src/pytask_r/plugin.py +++ b/src/pytask_r/plugin.py @@ -1,7 +1,7 @@ """Register hook specifications and implementations.""" from __future__ import annotations -from _pytask.config import hookimpl +from pytask import hookimpl from pytask_r import collect from pytask_r import config from pytask_r import execute diff --git a/src/pytask_r/serialization.py b/src/pytask_r/serialization.py new file mode 100644 index 0000000..9bb0499 --- /dev/null +++ b/src/pytask_r/serialization.py @@ -0,0 +1,65 @@ +"""This module contains the code to serialize keyword arguments to the task.""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from typing import Callable + +from pytask import Task + + +_HIDDEN_FOLDER = ".pytask" + + +SERIALIZER = {"json": {"serializer": json.dumps, "suffix": ".json"}} + + +try: + import yaml +except ImportError: # pragma: no cover + pass +else: + SERIALIZER["yaml"] = {"serializer": yaml.dump, "suffix": ".yaml"} + SERIALIZER["yml"] = {"serializer": yaml.dump, "suffix": ".yml"} + + +def create_path_to_serialized(task: Task, suffix: str) -> Path: + """Create path to serialized.""" + parent = task.path.parent + file_name = create_file_name(task, suffix) + path = parent.joinpath(_HIDDEN_FOLDER, file_name).with_suffix(suffix) + return path + + +def create_file_name(task: Task, suffix: str) -> str: + """Create the file name of the file containing the serialized kwargs. + + Some characters need to be escaped since they are not valid characters on file + systems. + + """ + return ( + task.short_name.replace("[", "_") + .replace("]", "_") + .replace("::", "_") + .replace(".", "_") + .replace("/", "_") + + suffix + ) + + +def serialize_keyword_arguments( + serializer: str | Callable[dict[str, Any], str], + path_to_serialized: Path, + kwargs: dict[str, Any], +) -> None: + if callable(serializer): + serializer_func = serializer + elif isinstance(serializer, str) and serializer in SERIALIZER: + serializer_func = SERIALIZER[serializer]["serializer"] + else: + raise ValueError(f"Serializer {serializer!r} is not known.") + + serialized = serializer_func(kwargs) + path_to_serialized.write_text(serialized) diff --git a/src/pytask_r/shared.py b/src/pytask_r/shared.py new file mode 100644 index 0000000..36026af --- /dev/null +++ b/src/pytask_r/shared.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Callable +from typing import Iterable +from typing import Sequence + + +_ERROR_MSG = """The old syntax for @pytask.mark.r was suddenly deprecated starting \ +with pytask-r v0.2 to provide a better user experience. Thank you for your \ +understanding! + +It is recommended to upgrade to the new syntax, so you enjoy all the benefits of v0.2 \ +of pytask and pytask-r which is among others access to 'depends_on' and 'produces', \ +and other kwargs inside the R script. + +You can find a manual here: \ +https://github.com/pytask-dev/pytask-r/blob/v0.2.0/README.md + +Upgrading can be as easy as rewriting your current task from + + @pytask.mark.r(["--option", "path_to_dependency.txt"]) + @pytask.mark.depends_on("script.R") + @pytask.mark.produces("out.csv") + def task_r(): + ... + +to + + @pytask.mark.r(script="script.r", options="--option") + @pytask.mark.depends_on("path_to_dependency.txt") + @pytask.mark.produces("out.csv") + def task_r(): + ... + +You can also fix the version of pytask and pytask-r to <0.2, so you do not have to \ +to upgrade. At the same time, you will not enjoy the improvements released with \ +version v0.2 of pytask and pytask-r. + +""" + + +def r( + *args, + script: str | Path = None, + options: str | Iterable[str] | None = None, + serializer: str | Callable[..., str] | str | None = None, + suffix: str | None = None, +) -> tuple[ + str | Path | None, + str | Iterable[str] | None, + str | Callable[..., str] | str | None, + str | None, +]: + """Parse input to the ``@pytask.mark.r`` decorator. + + Parameters + ---------- + script : Union[str, Path] + The path to the R script which is executed. + options : str | Iterable[str] + One or multiple command line options passed to Rscript. + serializer: Callable[Any, str] | None + A function to serialize data for the task which accepts a dictionary with all + the information. If the value is `None`, use either the value specified in the + configuration file under ``r_serializer`` or fall back to ``"json"``. + suffix: str | None + A suffix for the serialized file. If the value is `None`, use either the value + specified in the configuration file under ``r_suffix`` or fall back to + ``".json"``. + + """ + if args or script is None: + raise RuntimeError(_ERROR_MSG) + + options = [] if options is None else list(map(str, _to_list(options))) + return script, options, serializer, suffix + + +def _to_list(scalar_or_iter): + """Convert scalars and iterables to list. + + Parameters + ---------- + scalar_or_iter : str or list + + Returns + ------- + list + + Examples + -------- + >>> _to_list("a") + ['a'] + >>> _to_list(["b"]) + ['b'] + + """ + return ( + [scalar_or_iter] + if isinstance(scalar_or_iter, str) or not isinstance(scalar_or_iter, Sequence) + else list(scalar_or_iter) + ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py index 5e0288f..d593901 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,11 +5,31 @@ import pytest from click.testing import CliRunner + needs_rscript = pytest.mark.skipif( shutil.which("Rscript") is None, reason="R with Rscript needs to be installed." ) +parametrize_parse_code_serializer_suffix = pytest.mark.parametrize( + "parse_config_code, serializer, suffix", + [ + ( + "library(jsonlite); args <- commandArgs(trailingOnly=TRUE); " + "config <- read_json(args[length(args)])", + "json", + ".json", + ), + ( + "library(yaml); args <- commandArgs(trailingOnly=TRUE); " + "config <- read_yaml(args[length(args)])", + "yaml", + ".yaml", + ), + ], +) + + @pytest.fixture() def runner(): return CliRunner() diff --git a/tests/test_collect.py b/tests/test_collect.py index 4f9a0fd..fa78c14 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -1,131 +1,141 @@ from __future__ import annotations from contextlib import ExitStack as does_not_raise # noqa: N813 -from pathlib import Path import pytest -from _pytask.mark import Mark -from _pytask.nodes import FilePathNode -from pytask_r.collect import _get_node_from_dictionary -from pytask_r.collect import _merge_all_markers -from pytask_r.collect import _prepare_cmd_options -from pytask_r.collect import pytask_collect_task_teardown +from pytask import Mark +from pytask_r.collect import _parse_r_mark from pytask_r.collect import r - - -class DummyClass: - pass - - -def task_dummy(): - pass - - -@pytest.mark.unit -@pytest.mark.parametrize( - "r_args, expected", - [ - (None, []), - ("--some-option", ["--some-option"]), - (["--a", "--b"], ["--a", "--b"]), - ], -) -def test_r(r_args, expected): - options = r(r_args) - assert options == expected +from pytask_r.serialization import SERIALIZER @pytest.mark.unit @pytest.mark.parametrize( - "marks, expected", + "args, kwargs, expectation, expected", [ + ((), {}, pytest.raises(RuntimeError, match="The old syntax"), None), ( - [Mark("r", ("--a",), {}), Mark("r", ("--b",), {})], - Mark("r", ("--a", "--b"), {}), + ("-o"), + {"script": "script.r"}, + pytest.raises(RuntimeError, match="The old syntax"), + None, ), ( - [Mark("r", ("--a",), {}), Mark("r", (), {"r": "--b"})], - Mark("r", ("--a",), {"r": "--b"}), + (), + {"options": ("-o")}, + pytest.raises(RuntimeError, match="The old syntax"), + None, + ), + ( + (), + { + "script": "script.r", + "options": "--option", + "serializer": "json", + "suffix": ".json", + }, + does_not_raise(), + ("script.r", ["--option"], "json", ".json"), + ), + ( + (), + { + "script": "script.r", + "options": [1], + "serializer": "yaml", + "suffix": ".yaml", + }, + does_not_raise(), + ("script.r", ["1"], "yaml", ".yaml"), ), ], ) -def test_merge_all_markers(marks, expected): - task = DummyClass() - task.markers = marks - out = _merge_all_markers(task) - assert out == expected +def test_r(args, kwargs, expectation, expected): + with expectation: + result = r(*args, **kwargs) + assert result == expected @pytest.mark.unit @pytest.mark.parametrize( - "depends_on, produces, expectation", + "mark, default_options, default_serializer, default_suffix, expectation, expected", [ - (["script.r"], ["any_out.rds"], does_not_raise()), - (["script.R"], ["any_out.rds"], does_not_raise()), - (["script.txt"], ["any_out.rds"], pytest.raises(ValueError)), - (["input.rds", "script.r"], ["any_out.rds"], pytest.raises(ValueError)), - (["input.rds", "script.R"], ["any_out.rds"], pytest.raises(ValueError)), + ( + Mark("r", (), {}), + [], + None, + ".json", + pytest.raises(RuntimeError, match="The old syntax for @pytask.mark.r"), + Mark( + "r", + (), + { + "script": None, + "options": [], + "serializer": None, + "suffix": ".json", + }, + ), + ), + ( + Mark("r", ("-o"), {}), + [], + None, + ".json", + pytest.raises(RuntimeError, match="The old syntax for @pytask.mark.r"), + None, + ), + ( + Mark("r", (), {"script": "script.r"}), + [], + None, + ".json", + does_not_raise(), + Mark( + "r", + (), + { + "script": "script.r", + "options": [], + "serializer": None, + "suffix": ".json", + }, + ), + ), + ( + Mark( + "r", + (), + { + "script": "script.r", + "serializer": "json", + }, + ), + [], + None, + None, + does_not_raise(), + Mark( + "r", + (), + { + "script": "script.r", + "options": [], + "serializer": "json", + "suffix": SERIALIZER["json"]["suffix"], + }, + ), + ), ], ) -@pytest.mark.parametrize("r_source_key", ["source", "script"]) -def test_pytask_collect_task_teardown( - tmp_path, depends_on, produces, expectation, r_source_key +def test_parse_r_mark( + mark, + default_options, + default_serializer, + default_suffix, + expectation, + expected, ): - session = DummyClass() - session.config = {"r_source_key": r_source_key} - - task = DummyClass() - task.path = tmp_path / "task_dummy.py" - task.name = tmp_path.as_posix() + "task_dummy.py::task_dummy" - task.depends_on = { - i: FilePathNode.from_path(tmp_path / n) for i, n in enumerate(depends_on) - } - task.produces = { - i: FilePathNode.from_path(tmp_path / n) for i, n in enumerate(produces) - } - task.markers = [Mark("r", (), {})] - task.function = task_dummy - task.function.pytaskmark = task.markers - with expectation: - pytask_collect_task_teardown(session, task) - - -@pytest.mark.unit -@pytest.mark.parametrize( - "obj, key, expected", - [ - (1, "asds", 1), - (1, None, 1), - ({"a": 1}, "a", 1), - ({0: 1}, "a", 1), - ], -) -def test_get_node_from_dictionary(obj, key, expected): - result = _get_node_from_dictionary(obj, key) - assert result == expected - - -@pytest.mark.unit -@pytest.mark.parametrize( - "args", - [ - [], - ["a"], - ["a", "b"], - ], -) -@pytest.mark.parametrize("r_source_key", ["source", "script"]) -def test_prepare_cmd_options(args, r_source_key): - session = DummyClass() - session.config = {"r_source_key": r_source_key} - - node = DummyClass() - node.path = Path("script.r") - task = DummyClass() - task.depends_on = {r_source_key: node} - task.name = "task" - - result = _prepare_cmd_options(session, task, args) - - assert result == ["Rscript", "script.r", *args] + out = _parse_r_mark(mark, default_options, default_serializer, default_suffix) + assert out == expected diff --git a/tests/test_execute.py b/tests/test_execute.py index b9f98f1..291b704 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -1,53 +1,48 @@ from __future__ import annotations -import os import textwrap -from contextlib import ExitStack as does_not_raise # noqa: N813 +from pathlib import Path import pytest -from _pytask.mark import Mark -from conftest import needs_rscript +from pytask import cli +from pytask import ExitCode from pytask import main +from pytask import Mark +from pytask import Task from pytask_r.execute import pytask_execute_task_setup - -class DummyTask: - pass +from tests.conftest import needs_rscript +from tests.conftest import parametrize_parse_code_serializer_suffix @pytest.mark.unit -@pytest.mark.parametrize( - "found_r, expectation", - [ - (True, does_not_raise()), - (None, pytest.raises(RuntimeError)), - ], -) -def test_pytask_execute_task_setup(monkeypatch, found_r, expectation): +def test_pytask_execute_task_setup(monkeypatch): """Make sure that the task setup raises errors.""" # Act like r is installed since we do not test this. - monkeypatch.setattr( - "pytask_r.execute.shutil.which", lambda x: found_r # noqa: U100 - ) + monkeypatch.setattr("pytask_r.execute.shutil.which", lambda x: None) # noqa: U100 - task = DummyTask() - task.markers = [Mark("r", (), {})] + task = Task( + base_name="task_example", + path=Path(), + function=None, + markers=[Mark("r", (), {})], + ) - with expectation: + with pytest.raises(RuntimeError, match="Rscript is needed"): pytask_execute_task_setup(task) @needs_rscript @pytest.mark.end_to_end -@pytest.mark.parametrize( - "depends_on", - ["'script.r'", {"source": "script.r"}, {0: "script.r"}, {"script": "script.r"}], -) -def test_run_r_script(tmp_path, depends_on): +@parametrize_parse_code_serializer_suffix +@pytest.mark.parametrize("depends_on", ["'in_1.txt'", "['in_1.txt', 'in_2.txt']"]) +def test_run_r_script( + runner, tmp_path, parse_config_code, serializer, suffix, depends_on +): task_source = f""" import pytask - @pytask.mark.r + @pytask.mark.r(script="script.r", serializer="{serializer}", suffix="{suffix}") @pytask.mark.depends_on({depends_on}) @pytask.mark.produces("out.txt") def task_run_r_script(): @@ -55,44 +50,77 @@ def task_run_r_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - - r_script = """ - file_descriptor <- file("out.txt") + tmp_path.joinpath("in_1.txt").touch() + tmp_path.joinpath("in_2.txt").touch() + + r_script = f""" + {parse_config_code} + if(length(config["depends_on"]) <= 0){{ + stop("error message to print") + }} + file_descriptor <- file(config$produces) writeLines(c("So, so you think you can tell heaven from hell?"), file_descriptor) close(file_descriptor) """ tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) - if ( - isinstance(depends_on, dict) - and "source" not in depends_on - and 0 not in depends_on - ): - tmp_path.joinpath("pytask.ini").write_text("[pytask]\nr_source_key = script") + result = runner.invoke(cli, [tmp_path.as_posix()]) - os.chdir(tmp_path) - session = main({"paths": tmp_path}) + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("out.txt").exists() + + +@needs_rscript +@pytest.mark.end_to_end +@parametrize_parse_code_serializer_suffix +def test_run_r_script_w_task_decorator( + runner, tmp_path, parse_config_code, serializer, suffix +): + task_source = f""" + import pytask + + @pytask.mark.task + @pytask.mark.r(script="script.r", serializer="{serializer}", suffix="{suffix}") + @pytask.mark.produces("out.txt") + def run_r_script(): + pass + + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - assert session.exit_code == 0 + r_script = f""" + {parse_config_code} + file_descriptor <- file(config$produces) + writeLines(c("So, so you think you can tell heaven from hell?"), file_descriptor) + close(file_descriptor) + """ + tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK assert tmp_path.joinpath("out.txt").exists() +@needs_rscript @pytest.mark.end_to_end -def test_raise_error_if_rscript_is_not_found(tmp_path, monkeypatch): - task_source = """ +@parametrize_parse_code_serializer_suffix +def test_raise_error_if_rscript_is_not_found( + tmp_path, monkeypatch, parse_config_code, serializer, suffix +): + task_source = f""" import pytask - @pytask.mark.r - @pytask.mark.depends_on("script.r") + @pytask.mark.r(script="script.r", serializer="{serializer}", suffix="{suffix}") @pytask.mark.produces("out.txt") def task_run_r_script(): pass - """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - r_script = """ - file_descr <- file("out.txt") + r_script = f""" + {parse_config_code} + file_descr <- file(config$produces) writeLines(c("Birds flying high you know how I feel."), file_descr) close(file_descr) """ @@ -103,49 +131,90 @@ def task_run_r_script(): session = main({"paths": tmp_path}) - assert session.exit_code == 1 + assert session.exit_code == ExitCode.FAILED assert isinstance(session.execution_reports[0].exc_info[1], RuntimeError) @needs_rscript @pytest.mark.end_to_end -def test_run_r_script_w_saving_workspace(tmp_path): +@parametrize_parse_code_serializer_suffix +def test_run_r_script_w_saving_workspace( + runner, tmp_path, parse_config_code, serializer, suffix +): """Save workspace while executing the script.""" - task_source = """ + task_source = f""" import pytask - @pytask.mark.r("--save") - @pytask.mark.depends_on("script.r") + @pytask.mark.r( + script="script.r", + options="--save", + serializer="{serializer}", + suffix="{suffix}" + ) @pytask.mark.produces("out.txt") def task_run_r_script(): pass + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + r_script = f""" + {parse_config_code} + file_descr <- file(config$produces) + writeLines(c("Birds flying high you know how I feel."), file_descr) + close(file_descr) + """ + tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("out.txt").exists() + + +@needs_rscript +@pytest.mark.end_to_end +@parametrize_parse_code_serializer_suffix +def test_run_r_script_w_wrong_cmd_option( + runner, tmp_path, parse_config_code, serializer, suffix +): + """Save workspace while executing the script.""" + task_source = f""" + import pytask + + @pytask.mark.r( + script="script.r", + options="--wrong-flag", + serializer="{serializer}", + suffix="{suffix}" + ) + @pytask.mark.produces("out.txt") + def task_run_r_script(): + pass """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - r_script = """ - file_descr <- file("out.txt") + r_script = f""" + {parse_config_code} + file_descr <- file(config$produces) writeLines(c("Birds flying high you know how I feel."), file_descr) close(file_descr) """ tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) - os.chdir(tmp_path) - session = main({"paths": tmp_path}) + result = runner.invoke(cli, [tmp_path.as_posix()]) - assert session.exit_code == 0 + assert result.exit_code == ExitCode.OK assert tmp_path.joinpath("out.txt").exists() @needs_rscript @pytest.mark.end_to_end -def test_run_r_script_w_wrong_cmd_option(tmp_path): - """Apparently, Rscript simply discards wrong cmd options.""" +def test_run_r_script_w_custom_serializer(runner, tmp_path): task_source = """ import pytask + import json - @pytask.mark.r("--wrong-flag") - @pytask.mark.depends_on("script.r") + @pytask.mark.r(script="script.r", serializer=json.dumps, suffix=".json") @pytask.mark.produces("out.txt") def task_run_r_script(): pass @@ -154,13 +223,37 @@ def task_run_r_script(): tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) r_script = """ - file_descr <- file("out.txt") - writeLines(c("Birds flying high you know how I feel."), file_descr) - close(file_descr) + library(jsonlite) + args <- commandArgs(trailingOnly=TRUE) + config <- read_json(args[length(args)]) + file_descriptor <- file(config$produces) + writeLines(c("So, so you think you can tell heaven from hell?"), file_descriptor) + close(file_descriptor) """ tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) - os.chdir(tmp_path) - session = main({"paths": tmp_path}) + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("out.txt").exists() + + +@needs_rscript +@pytest.mark.end_to_end +def test_run_r_script_fails_w_multiple_markers(runner, tmp_path): + task_source = """ + import pytask + + @pytask.mark.r(script="script.r") + @pytask.mark.r(script="script.r") + @pytask.mark.produces("out.txt") + def task_run_r_script(): + pass + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + tmp_path.joinpath("script.r").touch() + + result = runner.invoke(cli, [tmp_path.as_posix()]) - assert session.exit_code == 0 + assert result.exit_code == ExitCode.COLLECTION_FAILED + assert "has multiple @pytask.mark.r marks" in result.output diff --git a/tests/test_normal_execution_w_plugin.py b/tests/test_normal_execution_w_plugin.py index ee19252..4a810eb 100644 --- a/tests/test_normal_execution_w_plugin.py +++ b/tests/test_normal_execution_w_plugin.py @@ -5,6 +5,7 @@ import pytest from pytask import cli +from pytask import ExitCode @pytest.mark.end_to_end @@ -35,4 +36,4 @@ def task_dummy(depends_on, produces): tmp_path.joinpath(dependency).touch() result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == 0 + assert result.exit_code == ExitCode.OK diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 8a04fb3..b7dbc6c 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -1,13 +1,14 @@ """Contains test which ensure that the plugin works with pytask-parallel.""" from __future__ import annotations -import os import textwrap import time import pytest -from conftest import needs_rscript from pytask import cli +from pytask import ExitCode + +from tests.conftest import needs_rscript try: import pytask_parallel # noqa: F401 @@ -22,43 +23,76 @@ ) +parametrize_parse_code_serializer_suffix = pytest.mark.parametrize( + "parse_config_code, serializer, suffix", + [ + ( + "library(jsonlite); args <- commandArgs(trailingOnly=TRUE); " + "config <- read_json(args[length(args)])", + "json", + ".json", + ) + ], +) + + @needs_rscript @pytest.mark.end_to_end -def test_parallel_parametrization_over_source_files(runner, tmp_path): +@parametrize_parse_code_serializer_suffix +def test_parallel_parametrization_over_source_files_w_parametrize( + runner, tmp_path, parse_config_code, serializer, suffix +): """Test parallelization over source files. - Same as in README.rst. + Same as in README.md. """ - os.chdir(tmp_path) - - source = """ + source = f""" import pytask - @pytask.mark.r @pytask.mark.parametrize( - "depends_on, produces", [("script_1.r", "1.rds"), ("script_2.r", "2.rds")] - ) + "r, content, produces", [ + ( + {{ + "script": "script_1.r", + "serializer": "{serializer}", + "suffix": "{suffix}", + }}, + 1, + "1.rds" + ), + ( + {{ + "script": "script_2.r", + "serializer": "{serializer}", + "suffix": "{suffix}", + }}, + 2, + "2.rds" + ) + ]) def task_execute_r_script(): pass """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) - r_script = """ - Sys.sleep(4) - saveRDS(1, file=paste0(1, ".rds")) + r_script = f""" + {parse_config_code} + Sys.sleep(6) + saveRDS(config$content, file=config$produces) """ tmp_path.joinpath("script_1.r").write_text(textwrap.dedent(r_script)) - r_script = """ - Sys.sleep(4) - saveRDS(2, file=paste0(2, ".rds")) + r_script = f""" + {parse_config_code} + Sys.sleep(6) + saveRDS(config$content, file=config$produces) """ tmp_path.joinpath("script_2.r").write_text(textwrap.dedent(r_script)) start = time.time() result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == 0 + assert result.exit_code == ExitCode.OK duration_normal = time.time() - start for name in ["1.rds", "2.rds"]: @@ -66,7 +100,7 @@ def task_execute_r_script(): start = time.time() result = runner.invoke(cli, [tmp_path.as_posix(), "-n", 2]) - assert result.exit_code == 0 + assert result.exit_code == ExitCode.OK duration_parallel = time.time() - start assert duration_parallel < duration_normal @@ -74,41 +108,148 @@ def task_execute_r_script(): @needs_rscript @pytest.mark.end_to_end -def test_parallel_parametrization_over_source_file(runner, tmp_path): - """Test parallelization over the same source file. +@parametrize_parse_code_serializer_suffix +def test_parallel_parametrization_over_source_files_w_loop( + runner, tmp_path, parse_config_code, serializer, suffix +): + """Test parallelization over source files. - Same as in README.rst. + Same as in README.md. """ - os.chdir(tmp_path) - - source = """ + source = f""" import pytask - from pathlib import Path - SRC = Path(__file__).parent + for i in range(1, 3): - @pytask.mark.depends_on("script.r") - @pytask.mark.parametrize("produces, r", [ - (SRC / "0.rds", (1, SRC / "0.rds")), (SRC / "1.rds", (1, SRC / "1.rds")) - ]) + @pytask.mark.task(kwargs={{"content": i}}) + @pytask.mark.r( + script="script_1.r", + serializer="{serializer}", + suffix="{suffix}" + ) + @pytask.mark.produces(f"{{i}}.rds") + def task_execute_r_script(): + pass + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + + r_script = f""" + {parse_config_code} + Sys.sleep(6) + saveRDS(config$content, file=config$produces) + """ + tmp_path.joinpath("script_1.r").write_text(textwrap.dedent(r_script)) + + r_script = f""" + {parse_config_code} + Sys.sleep(6) + saveRDS(config$content, file=config$produces) + """ + tmp_path.joinpath("script_2.r").write_text(textwrap.dedent(r_script)) + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + duration_normal = time.time() - start + + for name in ["1.rds", "2.rds"]: + tmp_path.joinpath(name).unlink() + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix(), "-n", 2]) + assert result.exit_code == ExitCode.OK + duration_parallel = time.time() - start + + assert duration_parallel < duration_normal + + +@needs_rscript +@pytest.mark.end_to_end +@parametrize_parse_code_serializer_suffix +def test_parallel_parametrization_over_source_file_w_parametrize( + runner, tmp_path, parse_config_code, serializer, suffix +): + """Test parallelization over the same source file. + + Same as in README.md. + + """ + source = f""" + import pytask + + @pytask.mark.r( + script="script.r", + serializer="{serializer}", + suffix="{suffix}", + ) + @pytask.mark.parametrize("produces, content", [("0.rds", 1), ("1.rds", 1)]) def task_execute_r_script(): pass """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) - r_script = """ - Sys.sleep(4) - args <- commandArgs(trailingOnly=TRUE) - number <- args[1] - produces <- args[2] - saveRDS(number, file=produces) + r_script = f""" + {parse_config_code} + Sys.sleep(6) + saveRDS(config$content, file=config$produces) + """ + tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + duration_normal = time.time() - start + + for name in ["0.rds", "1.rds"]: + tmp_path.joinpath(name).unlink() + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix(), "-n", 2]) + assert result.exit_code == ExitCode.OK + duration_parallel = time.time() - start + + assert duration_parallel < duration_normal + + +@needs_rscript +@pytest.mark.end_to_end +@parametrize_parse_code_serializer_suffix +def test_parallel_parametrization_over_source_file_w_loop( + runner, tmp_path, parse_config_code, serializer, suffix +): + """Test parallelization over the same source file. + + Same as in README.md. + + """ + source = f""" + import pytask + + for i in range(2): + + @pytask.mark.task(kwargs={{"content": i}}) + @pytask.mark.r( + script="script.r", + serializer="{serializer}", + suffix="{suffix}", + ) + @pytask.mark.produces(f"{{i}}.rds") + def execute_r_script(): + pass + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + + r_script = f""" + {parse_config_code} + Sys.sleep(6) + saveRDS(config$content, file=config$produces) """ tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) start = time.time() result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == 0 + assert result.exit_code == ExitCode.OK duration_normal = time.time() - start for name in ["0.rds", "1.rds"]: @@ -116,7 +257,7 @@ def task_execute_r_script(): start = time.time() result = runner.invoke(cli, [tmp_path.as_posix(), "-n", 2]) - assert result.exit_code == 0 + assert result.exit_code == ExitCode.OK duration_parallel = time.time() - start assert duration_parallel < duration_normal diff --git a/tests/test_parametrize.py b/tests/test_parametrize.py index 2406e56..075bc18 100644 --- a/tests/test_parametrize.py +++ b/tests/test_parametrize.py @@ -1,77 +1,184 @@ from __future__ import annotations -import os import textwrap import pytest -from conftest import needs_rscript -from pytask import main +from pytask import cli +from pytask import ExitCode + +from tests.conftest import needs_rscript + + +parametrize_parse_code_serializer_suffix = pytest.mark.parametrize( + "parse_config_code, serializer, suffix", + [ + ( + "library(jsonlite); args <- commandArgs(trailingOnly=TRUE); " + "config <- read_json(args[length(args)])", + "json", + ".json", + ) + ], +) @needs_rscript @pytest.mark.end_to_end -def test_parametrized_execution_of_r_script(tmp_path): - task_source = """ +@parametrize_parse_code_serializer_suffix +def test_parametrized_execution_of_r_script_w_parametrize( + runner, tmp_path, parse_config_code, serializer, suffix +): + task_source = f""" import pytask - @pytask.mark.r - @pytask.mark.parametrize("depends_on, produces", [ - ("script_1.r", "0.txt"), - ("script_2.r", "1.txt"), + @pytask.mark.parametrize("r, produces", [ + ( + {{ + "script": "script_1.r", + "serializer": "{serializer}", + "suffix": "{suffix}", + }}, + "0.txt" + ), + ( + {{ + "script": "script_2.r", + "serializer": "{serializer}", + "suffix": "{suffix}", + }}, + "1.txt" + ), ]) def task_run_r_script(): pass """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - for name, content, out in [ - ("script_1.r", "Cities breaking down on a camel's back", "0.txt"), - ("script_2.r", "They just have to go 'cause they don't know whack", "1.txt"), + for name, content in [ + ("script_1.r", "Cities breaking down on a camel's back"), + ("script_2.r", "They just have to go 'cause they don't know whack"), ]: r_script = f""" - file_descr <- file("{out}") + {parse_config_code} + file_descr <- file(config$produces) writeLines(c("{content}"), file_descr) close(file_descr) """ tmp_path.joinpath(name).write_text(textwrap.dedent(r_script)) - os.chdir(tmp_path) - session = main({"paths": tmp_path}) + result = runner.invoke(cli, [tmp_path.as_posix()]) - assert session.exit_code == 0 + assert result.exit_code == ExitCode.OK assert tmp_path.joinpath("0.txt").exists() assert tmp_path.joinpath("1.txt").exists() @needs_rscript @pytest.mark.end_to_end -def test_parametrize_r_options_and_product_paths(tmp_path): - task_source = """ +@parametrize_parse_code_serializer_suffix +def test_parametrized_execution_of_r_script_w_loop( + runner, tmp_path, parse_config_code, serializer, suffix +): + task_source = f""" import pytask - from pathlib import Path - SRC = Path(__file__).parent + for i in range(2): - @pytask.mark.depends_on("script.r") - @pytask.mark.parametrize("produces, r", [ - (SRC / "0.rds", (0, SRC / "0.rds")), (SRC / "1.rds", (1, SRC / "1.rds")) - ]) + @pytask.mark.task + @pytask.mark.r( + script=f"script_{{i + 1}}.r", + serializer="{serializer}", + suffix="{suffix}" + ) + @pytask.mark.produces(f"{{i}}.txt") + def task_run_r_script(): + pass + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + + for name, content in [ + ("script_1.r", "Cities breaking down on a camel's back"), + ("script_2.r", "They just have to go 'cause they don't know whack"), + ]: + r_script = f""" + {parse_config_code} + file_descr <- file(config$produces) + writeLines(c("{content}"), file_descr) + close(file_descr) + """ + tmp_path.joinpath(name).write_text(textwrap.dedent(r_script)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("0.txt").exists() + assert tmp_path.joinpath("1.txt").exists() + + +@needs_rscript +@pytest.mark.end_to_end +@parametrize_parse_code_serializer_suffix +def test_parametrize_r_options_and_product_paths_w_parametrize( + runner, tmp_path, parse_config_code, serializer, suffix +): + task_source = f""" + import pytask + + @pytask.mark.r( + script=f"script.r", + serializer="{serializer}", + suffix="{suffix}" + ) + @pytask.mark.parametrize("produces", ["0.rds", "1.rds"]) def task_execute_r_script(): pass """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - r_script = """ - args <- commandArgs(trailingOnly=TRUE) - number <- args[1] - produces <- args[2] - saveRDS(number, file=produces) + r_script = f""" + {parse_config_code} + saveRDS(config$number, file=config$produces) + """ + tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("0.rds").exists() + assert tmp_path.joinpath("1.rds").exists() + + +@needs_rscript +@pytest.mark.end_to_end +@parametrize_parse_code_serializer_suffix +def test_parametrize_r_options_and_product_paths_w_loop( + runner, tmp_path, parse_config_code, serializer, suffix +): + task_source = f""" + import pytask + + for i in range(2): + + @pytask.mark.task + @pytask.mark.r( + script=f"script.r", + serializer="{serializer}", + suffix="{suffix}" + ) + @pytask.mark.produces(f"{{i}}.rds") + def execute_r_script(): + pass + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + + r_script = f""" + {parse_config_code} + saveRDS(config$number, file=config$produces) """ tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) - os.chdir(tmp_path) - session = main({"paths": tmp_path}) + result = runner.invoke(cli, [tmp_path.as_posix()]) - assert session.exit_code == 0 + assert result.exit_code == ExitCode.OK assert tmp_path.joinpath("0.rds").exists() assert tmp_path.joinpath("1.rds").exists() diff --git a/tox.ini b/tox.ini index 3c48553..706acb7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = pytest, pre-commit +envlist = pytest skipsdist = True skip_missing_interpreters = True whitelist_externals = python @@ -11,6 +11,14 @@ basepython = python conda_deps = pytask >=0.1.0 pytask-parallel >=0.1.0 + pybaum >=0.1.1 + + # Optional dependencies. + pyyaml + r-base =4.1 + r-jsonlite + r-yaml + pytest pytest-cov pytest-xdist @@ -21,14 +29,6 @@ commands = pip install --no-deps -e . pytest {posargs} -[testenv:pre-commit] -deps = pre-commit -commands = pre-commit run --all-files - -[doc8] -ignore = D002, D004 -max-line-length = 89 - [flake8] docstring-convention = numpy ignore = @@ -48,7 +48,6 @@ filterwarnings = ignore: the imp module is deprecated in favour of importlib ignore: The (parser|symbol) module is deprecated and will be removed in future ignore: Using or importing the ABCs from 'collections' instead of from -junit_family = xunit2 markers = wip: Tests that are work-in-progress. unit: Flag for unit tests which target mainly a single function.