diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5221551..4e0b590 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,7 +48,7 @@ jobs: - name: Run unit tests and doctests. shell: bash -l {0} - run: tox -e pytest -- -m "unit or (not integration and not end_to_end)" --cov=./ --cov-report=xml -n auto + run: tox -e test -- -m "unit or (not integration and not end_to_end)" --cov=./ --cov-report=xml -n auto - name: Upload coverage report for unit tests and doctests. if: runner.os == 'Linux' && matrix.python-version == '3.10' @@ -57,7 +57,7 @@ jobs: - name: Run end-to-end tests. shell: bash -l {0} - run: tox -e pytest -- -m end_to_end --cov=./ --cov-report=xml -n auto + run: tox -e test -- -m end_to_end --cov=./ --cov-report=xml -n auto - name: Upload coverage reports of end-to-end tests. if: runner.os == 'Linux' && matrix.python-version == '3.10' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9de1e23..25d61c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,7 +46,7 @@ repos: additional_dependencies: [ attrs>=21.3.0, click, - pytask>=0.4.0, + pytask>=0.4.5, types-PyYAML, types-setuptools ] diff --git a/CHANGES.md b/CHANGES.md index 592d045..7e2b217 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,9 +5,10 @@ chronological order. Releases follow [semantic versioning](https://semver.org/) releases are available on [PyPI](https://pypi.org/project/pytask-r) and [Anaconda.org](https://anaconda.org/conda-forge/pytask-r). -## 0.4.1 - 2024-xx-xx +## 0.4.1 - 2024-04-20 - {pull}`46` modernizes the package. +- {pull}`47` fixes couple of issues raised in {issue}`45`. ## 0.4.0 - 2023-10-08 diff --git a/README.md b/README.md index d38ab1d..12b098f 100644 --- a/README.md +++ b/README.md @@ -31,34 +31,30 @@ You also need to have R installed and `Rscript` on your command line. Test it by the following on the command line ```console -$ Rscript --help +Rscript --help ``` -If an error is shown instead of a help page, you can install R with `conda` by choosing -either R or Microsoft R Open (MRO). Choose one of the two following commands. (See -[here](https://docs.anaconda.com/anaconda/user-guide/tasks/%20using-r-language) for -further explanation on Anaconda, R, and MRO.) +If an error is shown instead of a help page, you can install R with `conda`. ```console -$ conda install -c r r-base # For normal R. -$ conda install -c r mro-base # For MRO. +conda install -c conda-forge r-base ``` Or install install R from the official [R Project](https://www.r-project.org/). ## Usage -To create a task which runs a R script, define a task function with the `@pytask.mark.r` -decorator. The `script` keyword provides an absolute path or path relative to the task -module to the R script. +To create a task that runs an R script, define a task function with the `@mark.r` +decorator. The `script` keyword provides an absolute path or a path relative to the task +module. ```python -import pytask +from pathlib import Path +from pytask import mark -@pytask.mark.r(script="script.r") -@pytask.mark.produces("out.rds") -def task_run_r_script(): +@mark.r(script=Path("script.r")) +def task_run_r_script(produces: Path = Path("out.rds")): pass ``` @@ -68,10 +64,9 @@ more information. ### Dependencies and Products -Dependencies and products can be added as with a normal pytask task using the -`@pytask.mark.depends_on` and `@pytask.mark.produces` decorators. which is explained in -this -[tutorial](https://pytask-dev.readthedocs.io/en/stable/tutorials/defining_dependencies_products.html). +Dependencies and products can be added as usual. See this +[tutorial](https://pytask-dev.readthedocs.io/en/stable/tutorials/defining_dependencies_products.html) +for some help. ### Accessing dependencies and products in the script @@ -99,10 +94,13 @@ To parse the JSON file, you need to install You can also pass any other information to your script by using the `@task` decorator. ```python +from pathlib import Path +from pytask import mark, task + + @task(kwargs={"number": 1}) -@pytask.mark.r(script="script.r") -@pytask.mark.produces("out.rds") -def task_run_r_script(): +@mark.r(script=Path("script.r")) +def task_run_r_script(produces: Path = Path("out.rds")): pass ``` @@ -115,11 +113,11 @@ config$number # Is 1. ### Debugging In case a task throws an error, you might want to execute the script independently from -pytask. After a failed execution, you see the command which executed the R script in the +pytask. After a failed execution, you see the command that executed the R script in the report of the task. It looks roughly like this ```console -$ Rscript script.r /.pytask/task_py_task_example.json +Rscript script.r /.pytask/task_py_task_example.json ``` ### Command Line Arguments @@ -128,9 +126,8 @@ The decorator can be used to pass command line arguments to `Rscript`. See the f example. ```python -@pytask.mark.r(script="script.r", options="--vanilla") -@pytask.mark.produces("out.rds") -def task_run_r_script(): +@mark.r(script=Path("script.r"), options="--vanilla") +def task_run_r_script(produces: Path = Path("out.rds")): pass ``` @@ -146,9 +143,8 @@ different outputs. for i in range(2): @task - @pytask.mark.r(script=f"script_{i}.r") - @pytask.mark.produces(f"out_{i}.csv") - def task_execute_r_script(): + @mark.r(script=Path(f"script_{i}.r")) + def task_execute_r_script(produces: Path = Path(f"out_{i}.csv")): pass ``` @@ -159,9 +155,8 @@ If you want to pass different inputs to the same R script, pass these arguments for i in range(2): @task(kwargs={"i": i}) - @pytask.mark.r(script="script.r") - @pytask.mark.produces(f"output_{i}.csv") - def task_execute_r_script(): + @mark.r(script=Path("script.r")) + def task_execute_r_script(produces: Path = Path(f"output_{i}.csv")): pass ``` @@ -189,11 +184,11 @@ supports YAML (if PyYaml is installed). Use the `serializer` keyword arguments of the `@pytask.mark.r` decorator with ```python -@pytask.mark.r(script="script.r", serializer="yaml") +@mark.r(script=Path("script.r"), serializer="yaml") def task_example(): ... ``` -And in your R script use +And, in your R script use ```r library(yaml) @@ -203,8 +198,8 @@ config <- read_yaml(args[length(args)]) Note that the `YAML` package needs to be installed. -If you need a custom serializer, you can also provide any callable to `serializer` which -transforms data to a string. Use `suffix` to set the correct file ending. +If you need a custom serializer, you can also provide any callable `serializer` which +transforms data into a string. Use `suffix` to set the correct file ending. Here is a replication of the JSON example. @@ -212,13 +207,13 @@ Here is a replication of the JSON example. import json -@pytask.mark.r(script="script.r", serializer=json.dumps, suffix=".json") +@mark.r(script=Path("script.r"), serializer=json.dumps, suffix=".json") def task_example(): ... ``` ### Configuration -You can influence the default behavior of pytask-r with some configuration values. +You can influence the default behavior of pytask-r with configuration values. **`r_serializer`** diff --git a/pyproject.toml b/pyproject.toml index 4e6fd96..05d0ce2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ classifiers = [ "Programming Language :: R", ] requires-python = ">=3.8" -dependencies = ["click", "pluggy>=1.0.0", "pytask>=0.4.0"] +dependencies = ["click", "pluggy>=1.0.0", "pytask>=0.4.5"] dynamic = ["version"] [[project.authors]] diff --git a/src/pytask_r/collect.py b/src/pytask_r/collect.py index d2a3e3b..0cf2a2f 100644 --- a/src/pytask_r/collect.py +++ b/src/pytask_r/collect.py @@ -66,11 +66,11 @@ def pytask_collect_task( default_serializer=session.config["r_serializer"], default_suffix=session.config["r_suffix"], ) - script, options, _, suffix = r(**marks[0].kwargs) + script, options, _, suffix = r(**mark.kwargs) obj.pytask_meta.markers.append(mark) - # Collect the nodes in @pytask.mark.julia and validate them. + # Collect the nodes in @pytask.mark.r and validate them. path_nodes = Path.cwd() if path is None else path.parent if isinstance(script, str): @@ -93,10 +93,13 @@ def pytask_collect_task( ), ) - if not (isinstance(script_node, PathNode) and script_node.path.suffix == ".r"): + if not ( + isinstance(script_node, PathNode) + and script_node.path.suffix in (".r", ".R") + ): msg = ( "The 'script' keyword of the @pytask.mark.r decorator must point " - f"to Julia file with the .r suffix, but it is {script_node}." + f"to an R file with the .r or .R extension, but it is {script_node}." ) raise ValueError(msg) @@ -169,7 +172,7 @@ def _parse_r_mark( default_serializer: str, default_suffix: str, ) -> Mark: - """Parse a Julia mark.""" + """Parse an R mark.""" script, options, serializer, suffix = r(**mark.kwargs) parsed_kwargs = {} diff --git a/src/pytask_r/config.py b/src/pytask_r/config.py index a9eca85..603454c 100644 --- a/src/pytask_r/config.py +++ b/src/pytask_r/config.py @@ -20,7 +20,7 @@ def pytask_parse_config(config: dict[str, Any]) -> None: f"{list(SERIALIZERS)}." ) raise ValueError(msg) - config["r_suffix"] = config.get("r_suffix", "") + config["r_suffix"] = config.get("r_suffix", ".json") config["r_options"] = _parse_value_or_whitespace_option(config.get("r_options")) diff --git a/src/pytask_r/execute.py b/src/pytask_r/execute.py index dd80881..8936c32 100644 --- a/src/pytask_r/execute.py +++ b/src/pytask_r/execute.py @@ -44,11 +44,11 @@ def collect_keyword_arguments(task: PTask) -> dict[str, Any]: # Remove all kwargs from the task so that they are not passed to the function. kwargs: dict[str, Any] = { **tree_map( # type: ignore[dict-item] - lambda x: str(x.path) if isinstance(x, PPathNode) else str(x.value), + lambda x: str(x.path) if isinstance(x, PPathNode) else x.value, task.depends_on, ), **tree_map( # type: ignore[dict-item] - lambda x: str(x.path) if isinstance(x, PPathNode) else str(x.value), + lambda x: str(x.path) if isinstance(x, PPathNode) else x.value, task.produces, ), } diff --git a/tests/test_execute.py b/tests/test_execute.py index 3a51c74..e0bdb75 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -40,14 +40,12 @@ def test_run_r_script( # noqa: PLR0913 runner, tmp_path, parse_config_code, serializer, suffix, depends_on ): task_source = f""" - import pytask - - @pytask.mark.r(script="script.r", serializer="{serializer}", suffix="{suffix}") - @pytask.mark.depends_on({depends_on}) - @pytask.mark.produces("out.txt") - def task_run_r_script(): - pass + from pathlib import Path + from pytask import mark + @mark.r(script="script.r", serializer="{serializer}", suffix="{suffix}") + @mark.depends_on({depends_on}) + def task_run_r_script(produces = Path("out.txt")): ... """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) tmp_path.joinpath("in_1.txt").touch() @@ -56,7 +54,7 @@ def task_run_r_script(): r_script = f""" {parse_config_code} if(length(config["depends_on"]) <= 0){{ - stop("error message to print") + stop("error message to print") # noqa: T201 }} file_descriptor <- file(config$produces) writeLines(c("So, so you think you can tell heaven from hell?"), file_descriptor) @@ -77,14 +75,12 @@ def test_run_r_script_w_task_decorator( runner, tmp_path, parse_config_code, serializer, suffix ): task_source = f""" - import pytask - - @pytask.mark.task - @pytask.mark.r(script="script.r", serializer="{serializer}", suffix="{suffix}") - @pytask.mark.produces("out.txt") - def run_r_script(): - pass + from pytask import task, mark + @task + @mark.r(script="script.r", serializer="{serializer}", suffix="{suffix}") + @mark.produces("out.txt") + def run_r_script(): ... """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) @@ -113,8 +109,7 @@ def test_raise_error_if_rscript_is_not_found( @pytask.mark.r(script="script.r", serializer="{serializer}", suffix="{suffix}") @pytask.mark.produces("out.txt") - def task_run_r_script(): - pass + def task_run_r_script(): ... """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) @@ -152,8 +147,7 @@ def test_run_r_script_w_saving_workspace( suffix="{suffix}" ) @pytask.mark.produces("out.txt") - def task_run_r_script(): - pass + def task_run_r_script(): ... """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) @@ -188,8 +182,7 @@ def test_run_r_script_w_wrong_cmd_option( suffix="{suffix}" ) @pytask.mark.produces("out.txt") - def task_run_r_script(): - pass + def task_run_r_script(): ... """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) @@ -216,8 +209,7 @@ def test_run_r_script_w_custom_serializer(runner, tmp_path): @pytask.mark.r(script="script.r", serializer=json.dumps, suffix=".json") @pytask.mark.produces("out.txt") - def task_run_r_script(): - pass + def task_run_r_script(): ... """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) @@ -247,8 +239,7 @@ def test_run_r_script_fails_w_multiple_markers(runner, tmp_path): @pytask.mark.r(script="script.r") @pytask.mark.r(script="script.r") @pytask.mark.produces("out.txt") - def task_run_r_script(): - pass + def task_run_r_script(): ... """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) tmp_path.joinpath("script.r").touch() @@ -257,3 +248,61 @@ def task_run_r_script(): assert result.exit_code == ExitCode.COLLECTION_FAILED assert "has multiple @pytask.mark.r marks" in result.output + + +@needs_rscript +@pytest.mark.end_to_end() +def test_run_r_script_with_capital_extension(runner, tmp_path): + task_source = """ + import pytask + + @pytask.mark.r(script="script.R") + @pytask.mark.produces("out.txt") + def task_run_r_script(): ... + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + + r_script = """ + library(jsonlite) + args <- commandArgs(trailingOnly=TRUE) + config <- read_json(args[length(args)]) + file_descriptor <- file(config$produces) + writeLines(c("So, so you think you can tell heaven from hell?"), file_descriptor) + close(file_descriptor) + """ + tmp_path.joinpath("script.R").write_text(textwrap.dedent(r_script)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("out.txt").exists() + + +@needs_rscript +@pytest.mark.end_to_end() +@parametrize_parse_code_serializer_suffix +def test_run_r_script_w_nested_inputs( + runner, tmp_path, parse_config_code, serializer, suffix +): + task_source = f""" + from pytask import mark, task + + @task(kwargs={{"content": {{"first": "Hello, ", "second": "World!"}}}}) + @mark.r(script="script.r", serializer="{serializer}", suffix="{suffix}") + @mark.produces("out.txt") + def task_run_r_script(): ... + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + + r_script = f""" + {parse_config_code} + file_descriptor <- file(config$produces) + writeLines(c(config$content$first, config$content$second), file_descriptor) + close(file_descriptor) + """ + tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("out.txt").read_text() == "Hello, \nWorld!\n" diff --git a/tox.ini b/tox.ini index 1721656..4e09674 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = pytest +envlist = test [testenv] # Cannot use package = editable, because tox-conda does not support tox v4.