From e98cfadccbb5e7a429dbbc48b9bb304740836ccd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 19:24:17 +0000 Subject: [PATCH 1/5] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.4.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.4.0...v4.6.0) - [github.com/psf/black: 23.9.1 → 24.4.0](https://github.com/psf/black/compare/23.9.1...24.4.0) - [github.com/astral-sh/ruff-pre-commit: v0.0.292 → v0.3.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.292...v0.3.7) - [github.com/dosisod/refurb: v1.21.0 → v2.0.0](https://github.com/dosisod/refurb/compare/v1.21.0...v2.0.0) - [github.com/econchick/interrogate: 1.5.0 → 1.7.0](https://github.com/econchick/interrogate/compare/1.5.0...1.7.0) - [github.com/pre-commit/mirrors-mypy: v1.5.1 → v1.9.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.5.1...v1.9.0) --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d8d6d6..55e26d3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: check-added-large-files args: ['--maxkb=100'] @@ -27,20 +27,20 @@ repos: hooks: - id: setup-cfg-fmt - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 24.4.0 hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.292 + rev: v0.3.7 hooks: - id: ruff - repo: https://github.com/dosisod/refurb - rev: v1.21.0 + rev: v2.0.0 hooks: - id: refurb args: [--ignore, FURB126] - repo: https://github.com/econchick/interrogate - rev: 1.5.0 + rev: 1.7.0 hooks: - id: interrogate args: [-v, --fail-under=40, src, tests] @@ -58,7 +58,7 @@ repos: hooks: - id: codespell - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.5.1' + rev: 'v1.9.0' hooks: - id: mypy additional_dependencies: [ From 08d30569a0beb01f34dd70c41a1df73306f907d5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 19:26:03 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 6 ++---- src/pytask_r/__init__.py | 1 + src/pytask_r/collect.py | 1 + src/pytask_r/config.py | 1 + src/pytask_r/execute.py | 1 + src/pytask_r/plugin.py | 1 + src/pytask_r/serialization.py | 1 + src/pytask_r/shared.py | 1 + tests/test_normal_execution_w_plugin.py | 1 + tests/test_parallel.py | 1 + 10 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9c29ed1..d38ab1d 100644 --- a/README.md +++ b/README.md @@ -190,8 +190,7 @@ Use the `serializer` keyword arguments of the `@pytask.mark.r` decorator with ```python @pytask.mark.r(script="script.r", serializer="yaml") -def task_example(): - ... +def task_example(): ... ``` And in your R script use @@ -214,8 +213,7 @@ import json @pytask.mark.r(script="script.r", serializer=json.dumps, suffix=".json") -def task_example(): - ... +def task_example(): ... ``` ### Configuration diff --git a/src/pytask_r/__init__.py b/src/pytask_r/__init__.py index 917f1d8..ab442a8 100644 --- a/src/pytask_r/__init__.py +++ b/src/pytask_r/__init__.py @@ -1,4 +1,5 @@ """This module contains the main namespace of pytask-r.""" + from __future__ import annotations try: diff --git a/src/pytask_r/collect.py b/src/pytask_r/collect.py index f79bdb7..9f3a0b7 100644 --- a/src/pytask_r/collect.py +++ b/src/pytask_r/collect.py @@ -1,4 +1,5 @@ """Collect tasks.""" + from __future__ import annotations import subprocess diff --git a/src/pytask_r/config.py b/src/pytask_r/config.py index f107589..792b162 100644 --- a/src/pytask_r/config.py +++ b/src/pytask_r/config.py @@ -1,4 +1,5 @@ """Configure pytask.""" + from __future__ import annotations from typing import Any diff --git a/src/pytask_r/execute.py b/src/pytask_r/execute.py index ce6d427..e247975 100644 --- a/src/pytask_r/execute.py +++ b/src/pytask_r/execute.py @@ -1,4 +1,5 @@ """Execute tasks.""" + from __future__ import annotations import shutil diff --git a/src/pytask_r/plugin.py b/src/pytask_r/plugin.py index ec6efc8..de00bfa 100644 --- a/src/pytask_r/plugin.py +++ b/src/pytask_r/plugin.py @@ -1,4 +1,5 @@ """Register hook specifications and implementations.""" + from __future__ import annotations from pluggy import PluginManager diff --git a/src/pytask_r/serialization.py b/src/pytask_r/serialization.py index 6b23842..bd47351 100644 --- a/src/pytask_r/serialization.py +++ b/src/pytask_r/serialization.py @@ -1,4 +1,5 @@ """This module contains the code to serialize keyword arguments to the task.""" + from __future__ import annotations import json diff --git a/src/pytask_r/shared.py b/src/pytask_r/shared.py index 49d29f1..00ff27e 100644 --- a/src/pytask_r/shared.py +++ b/src/pytask_r/shared.py @@ -1,4 +1,5 @@ """This module contains shared functions.""" + from __future__ import annotations from pathlib import Path diff --git a/tests/test_normal_execution_w_plugin.py b/tests/test_normal_execution_w_plugin.py index 78d67cc..53a0555 100644 --- a/tests/test_normal_execution_w_plugin.py +++ b/tests/test_normal_execution_w_plugin.py @@ -1,4 +1,5 @@ """Contains tests which do not require the plugin and ensure normal execution.""" + from __future__ import annotations import textwrap diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 587c649..e659f7b 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -1,4 +1,5 @@ """Contains test which ensure that the plugin works with pytask-parallel.""" + from __future__ import annotations import textwrap From ef5f8fd5f827efb1078dfaed401c82febcf49fc2 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 19 Apr 2024 00:06:22 +0200 Subject: [PATCH 3/5] Fix. --- .pre-commit-config.yaml | 18 ++---------- pyproject.toml | 37 +++++++------------------ src/pytask_r/__init__.py | 2 +- src/pytask_r/collect.py | 29 +++++++++++-------- src/pytask_r/config.py | 7 +++-- src/pytask_r/execute.py | 12 ++++---- src/pytask_r/plugin.py | 7 ++++- src/pytask_r/serialization.py | 13 ++++----- src/pytask_r/shared.py | 7 +++-- tests/conftest.py | 5 ++-- tests/test_execute.py | 4 +-- tests/test_normal_execution_w_plugin.py | 2 +- tests/test_parallel.py | 2 +- tests/test_parametrize.py | 2 +- 14 files changed, 65 insertions(+), 82 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55e26d3..fef43a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,33 +17,19 @@ repos: - id: python-no-log-warn - id: python-use-type-annotations - id: text-unicode-replacement-char -- repo: https://github.com/asottile/reorder-python-imports - rev: v3.12.0 - hooks: - - id: reorder-python-imports - args: [--py37-plus, --add-import, 'from __future__ import annotations'] - repo: https://github.com/asottile/setup-cfg-fmt rev: v2.5.0 hooks: - id: setup-cfg-fmt -- repo: https://github.com/psf/black - rev: 24.4.0 - hooks: - - id: black - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.3.7 hooks: - id: ruff + - id: ruff-format - repo: https://github.com/dosisod/refurb rev: v2.0.0 hooks: - id: refurb - args: [--ignore, FURB126] -- repo: https://github.com/econchick/interrogate - rev: 1.7.0 - hooks: - - id: interrogate - args: [-v, --fail-under=40, src, tests] - repo: https://github.com/executablebooks/mdformat rev: 0.7.17 hooks: @@ -74,7 +60,7 @@ repos: hooks: - id: check-manifest args: [--no-build-isolation] - additional_dependencies: [setuptools-scm, toml] + additional_dependencies: [setuptools-scm, wheel, toml] - repo: meta hooks: - id: check-hooks-apply diff --git a/pyproject.toml b/pyproject.toml index 922ea07..060dd09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,11 +2,9 @@ requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.0"] build-backend = "setuptools.build_meta" - [tool.setuptools_scm] write_to = "src/pytask_r/_version.py" - [tool.mypy] files = ["src", "tests"] check_untyped_defs = true @@ -17,56 +15,41 @@ no_implicit_optional = true warn_redundant_casts = true warn_unused_ignores = true - [[tool.mypy.overrides]] module = "tests.*" disallow_untyped_defs = false ignore_errors = true - [tool.ruff] target-version = "py38" -select = ["ALL"] fix = true +unsafe-fixes = true + +[tool.ruff.lint] extend-ignore = [ - "TCH", - "TRY", - # Numpy docstyle - "D107", - "D203", - "D212", - "D213", - "D402", - "D413", - "D415", - "D416", - "D417", - # Others. - "D404", # Do not start module docstring with "This". - "RET504", # unnecessary variable assignment before return. "S101", # raise errors for asserts. "B905", # strict parameter for zip that was implemented in py310. - "I", # ignore isort "ANN101", # type annotating self "ANN102", # type annotating cls "FBT", # flake8-boolean-trap "EM", # flake8-errmsg "ANN401", # flake8-annotate typing.Any "PD", # pandas-vet - "COM812", # trailing comma missing, but black takes care of that + "COM812", # Comply with ruff-format + "ISC001", # Comply with ruff-format ] +select = ["ALL"] - -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "tests/*" = ["D", "ANN"] - -[tool.ruff.pydocstyle] +[tool.ruff.lint.pydocstyle] convention = "numpy" +[tool.ruff.lint.isort] +force-single-line = true [tool.pytest.ini_options] -# Do not add src since it messes with the loading of pytask-parallel as a plugin. testpaths = ["tests"] markers = [ "wip: Tests that are work-in-progress.", diff --git a/src/pytask_r/__init__.py b/src/pytask_r/__init__.py index ab442a8..8c31e12 100644 --- a/src/pytask_r/__init__.py +++ b/src/pytask_r/__init__.py @@ -1,4 +1,4 @@ -"""This module contains the main namespace of pytask-r.""" +"""Contains the main namespace of pytask-r.""" from __future__ import annotations diff --git a/src/pytask_r/collect.py b/src/pytask_r/collect.py index 9f3a0b7..d2a3e3b 100644 --- a/src/pytask_r/collect.py +++ b/src/pytask_r/collect.py @@ -7,27 +7,31 @@ from pathlib import Path from typing import Any -from pytask import has_mark -from pytask import hookimpl -from pytask import is_task_function from pytask import Mark from pytask import NodeInfo -from pytask import parse_dependencies_from_task_function -from pytask import parse_products_from_task_function from pytask import PathNode from pytask import PTask from pytask import PythonNode -from pytask import remove_marks from pytask import Session from pytask import Task from pytask import TaskWithoutPath -from pytask_r.serialization import create_path_to_serialized +from pytask import has_mark +from pytask import hookimpl +from pytask import is_task_function +from pytask import parse_dependencies_from_task_function +from pytask import parse_products_from_task_function +from pytask import remove_marks + from pytask_r.serialization import SERIALIZERS +from pytask_r.serialization import create_path_to_serialized from pytask_r.shared import r def run_r_script( - _script: Path, _options: list[str], _serialized: Path, **kwargs: Any # noqa: ARG001 + _script: Path, + _options: list[str], + _serialized: Path, + **kwargs: Any, # noqa: ARG001 ) -> None: """Run an R script.""" cmd = ["Rscript", _script.as_posix(), *_options, str(_serialized)] @@ -50,10 +54,11 @@ def pytask_collect_task( # Parse @pytask.mark.r decorator. obj, marks = remove_marks(obj, "r") if len(marks) > 1: - raise ValueError( + msg = ( f"Task {name!r} has multiple @pytask.mark.r marks, but only one is " "allowed." ) + raise ValueError(msg) mark = _parse_r_mark( mark=marks[0], @@ -89,10 +94,11 @@ def pytask_collect_task( ) if not (isinstance(script_node, PathNode) and script_node.path.suffix == ".r"): - raise ValueError( + msg = ( "The 'script' keyword of the @pytask.mark.r decorator must point " f"to Julia file with the .r suffix, but it is {script_node}." ) + raise ValueError(msg) options_node = session.hook.pytask_collect_node( session=session, @@ -182,5 +188,4 @@ def _parse_r_mark( ) parsed_kwargs["suffix"] = suffix or proposed_suffix # type: ignore[assignment] - mark = Mark("r", (), parsed_kwargs) - return mark + return Mark("r", (), parsed_kwargs) diff --git a/src/pytask_r/config.py b/src/pytask_r/config.py index 792b162..a9eca85 100644 --- a/src/pytask_r/config.py +++ b/src/pytask_r/config.py @@ -5,6 +5,7 @@ from typing import Any from pytask import hookimpl + from pytask_r.serialization import SERIALIZERS @@ -14,10 +15,11 @@ def pytask_parse_config(config: dict[str, Any]) -> None: config["markers"]["r"] = "Tasks which are executed with Rscript." config["r_serializer"] = config.get("r_serializer", "json") if config["r_serializer"] not in SERIALIZERS: - raise ValueError( + msg = ( f"'r_serializer' is {config['r_serializer']} and not one of " f"{list(SERIALIZERS)}." ) + raise ValueError(msg) config["r_suffix"] = config.get("r_suffix", "") config["r_options"] = _parse_value_or_whitespace_option(config.get("r_options")) @@ -28,4 +30,5 @@ def _parse_value_or_whitespace_option(value: Any) -> list[str] | None: return None if isinstance(value, list): return list(map(str, value)) - raise ValueError(f"'r_options' is {value} and not a list.") + msg = f"'r_options' is {value} and not a list." + raise ValueError(msg) diff --git a/src/pytask_r/execute.py b/src/pytask_r/execute.py index e247975..fabb9f9 100644 --- a/src/pytask_r/execute.py +++ b/src/pytask_r/execute.py @@ -5,12 +5,13 @@ import shutil from typing import Any -from pytask import get_marks -from pytask import hookimpl from pytask import PPathNode from pytask import PTask from pytask import PythonNode +from pytask import get_marks +from pytask import hookimpl from pytask.tree_util import tree_map + from pytask_r.serialization import serialize_keyword_arguments from pytask_r.shared import r @@ -21,9 +22,10 @@ def pytask_execute_task_setup(task: PTask) -> None: marks = get_marks(task, "r") if marks: if shutil.which("Rscript") is None: - raise RuntimeError( + msg = ( "Rscript is needed to run R scripts, but it is not found on your PATH." ) + raise RuntimeError(msg) assert len(marks) == 1 @@ -32,9 +34,9 @@ def pytask_execute_task_setup(task: PTask) -> None: assert suffix serialized_node: PythonNode = task.depends_on["_serialized"] # type: ignore[assignment] - serialized_node.value.parent.mkdir(parents=True, exist_ok=True) + serialized_node.value.parent.mkdir(parents=True, exist_ok=True) # type: ignore[union-attr] kwargs = collect_keyword_arguments(task) - serialize_keyword_arguments(serializer, serialized_node.value, kwargs) + serialize_keyword_arguments(serializer, serialized_node.value, kwargs) # type: ignore[arg-type] def collect_keyword_arguments(task: PTask) -> dict[str, Any]: diff --git a/src/pytask_r/plugin.py b/src/pytask_r/plugin.py index de00bfa..7281a99 100644 --- a/src/pytask_r/plugin.py +++ b/src/pytask_r/plugin.py @@ -2,12 +2,17 @@ from __future__ import annotations -from pluggy import PluginManager +from typing import TYPE_CHECKING + from pytask import hookimpl + from pytask_r import collect from pytask_r import config from pytask_r import execute +if TYPE_CHECKING: + from pluggy import PluginManager + @hookimpl def pytask_add_hooks(pm: PluginManager) -> None: diff --git a/src/pytask_r/serialization.py b/src/pytask_r/serialization.py index bd47351..e95bf35 100644 --- a/src/pytask_r/serialization.py +++ b/src/pytask_r/serialization.py @@ -1,4 +1,4 @@ -"""This module contains the code to serialize keyword arguments to the task.""" +"""Contains the code to serialize keyword arguments to the task.""" from __future__ import annotations @@ -10,7 +10,6 @@ from pytask import PTask from pytask import PTaskWithPath - _HIDDEN_FOLDER = ".pytask/pytask-r" @@ -30,8 +29,7 @@ def create_path_to_serialized(task: PTask, suffix: str) -> Path: """Create path to serialized.""" parent = task.path.parent if isinstance(task, PTaskWithPath) else Path.cwd() file_name = create_file_name(task, suffix) - path = parent.joinpath(_HIDDEN_FOLDER, file_name).with_suffix(suffix) - return path + return parent.joinpath(_HIDDEN_FOLDER, file_name).with_suffix(suffix) def create_file_name(task: PTask, suffix: str) -> str: @@ -60,11 +58,10 @@ def serialize_keyword_arguments( if callable(serializer): serializer_func = serializer elif isinstance(serializer, str) and serializer in SERIALIZERS: - serializer_func = SERIALIZERS[serializer][ - "serializer" - ] # type: ignore[assignment] + serializer_func = SERIALIZERS[serializer]["serializer"] # type: ignore[assignment] else: - raise ValueError(f"Serializer {serializer!r} is not known.") + msg = f"Serializer {serializer!r} is not known." + raise ValueError(msg) serialized = serializer_func(kwargs) path_to_serialized.write_text(serialized) diff --git a/src/pytask_r/shared.py b/src/pytask_r/shared.py index 00ff27e..03d4618 100644 --- a/src/pytask_r/shared.py +++ b/src/pytask_r/shared.py @@ -1,13 +1,16 @@ -"""This module contains shared functions.""" +"""Contains shared functions.""" from __future__ import annotations -from pathlib import Path +from typing import TYPE_CHECKING from typing import Any from typing import Callable from typing import Iterable from typing import Sequence +if TYPE_CHECKING: + from pathlib import Path + def r( *, diff --git a/tests/conftest.py b/tests/conftest.py index 64daf97..47da47c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,6 @@ import pytest from click.testing import CliRunner - needs_rscript = pytest.mark.skipif( shutil.which("Rscript") is None, reason="R with Rscript needs to be installed." ) @@ -37,7 +36,7 @@ class SysPathsSnapshot: """A snapshot for sys.path.""" def __init__(self) -> None: - self.__saved = list(sys.path), list(sys.meta_path) + self.__saved = sys.path.copy(), sys.meta_path.copy() def restore(self) -> None: sys.path[:], sys.meta_path[:] = self.__saved @@ -48,7 +47,7 @@ class SysModulesSnapshot: def __init__(self, preserve: Callable[[str], bool] | None = None) -> None: self.__preserve = preserve - self.__saved = dict(sys.modules) + self.__saved = sys.modules.copy() def restore(self) -> None: if self.__preserve: diff --git a/tests/test_execute.py b/tests/test_execute.py index dc67b85..3a51c74 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -4,11 +4,11 @@ from pathlib import Path import pytest -from pytask import build -from pytask import cli from pytask import ExitCode from pytask import Mark from pytask import Task +from pytask import build +from pytask import cli from pytask_r.execute import pytask_execute_task_setup from tests.conftest import needs_rscript diff --git a/tests/test_normal_execution_w_plugin.py b/tests/test_normal_execution_w_plugin.py index 53a0555..3c8813f 100644 --- a/tests/test_normal_execution_w_plugin.py +++ b/tests/test_normal_execution_w_plugin.py @@ -5,8 +5,8 @@ import textwrap import pytest -from pytask import cli from pytask import ExitCode +from pytask import cli @pytest.mark.end_to_end() diff --git a/tests/test_parallel.py b/tests/test_parallel.py index e659f7b..a6754a4 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -5,8 +5,8 @@ import textwrap import pytest -from pytask import cli from pytask import ExitCode +from pytask import cli from tests.conftest import needs_rscript from tests.conftest import parametrize_parse_code_serializer_suffix diff --git a/tests/test_parametrize.py b/tests/test_parametrize.py index 1fe4333..6e9cb94 100644 --- a/tests/test_parametrize.py +++ b/tests/test_parametrize.py @@ -3,8 +3,8 @@ import textwrap import pytest -from pytask import cli from pytask import ExitCode +from pytask import cli from tests.conftest import needs_rscript from tests.conftest import parametrize_parse_code_serializer_suffix From 4c6cffdd0c5fb92600d65db789c5e772bac0949f Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 19 Apr 2024 00:10:47 +0200 Subject: [PATCH 4/5] fix. --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 47da47c..c4dc905 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,7 @@ import sys from contextlib import contextmanager from typing import Callable - +from pytask import storage import pytest from click.testing import CliRunner @@ -86,6 +86,7 @@ def _restore_sys_path_and_module_after_test_execution(): class CustomCliRunner(CliRunner): def invoke(self, *args, **kwargs): """Restore sys.path and sys.modules after an invocation.""" + storage.create() with restore_sys_path_and_module_after_test_execution(): return super().invoke(*args, **kwargs) From 3929baa15328eb8e6c7e6a830f47b8f0c8e768e2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Apr 2024 22:11:28 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index c4dc905..c8d16f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,9 +4,10 @@ import sys from contextlib import contextmanager from typing import Callable -from pytask import storage + import pytest from click.testing import CliRunner +from pytask import storage needs_rscript = pytest.mark.skipif( shutil.which("Rscript") is None, reason="R with Rscript needs to be installed."