diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d8d6d6..fef43a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: check-added-large-files args: ['--maxkb=100'] @@ -17,33 +17,19 @@ repos: - id: python-no-log-warn - id: python-use-type-annotations - id: text-unicode-replacement-char -- repo: https://github.com/asottile/reorder-python-imports - rev: v3.12.0 - hooks: - - id: reorder-python-imports - args: [--py37-plus, --add-import, 'from __future__ import annotations'] - repo: https://github.com/asottile/setup-cfg-fmt rev: v2.5.0 hooks: - id: setup-cfg-fmt -- repo: https://github.com/psf/black - rev: 23.9.1 - hooks: - - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.292 + rev: v0.3.7 hooks: - id: ruff + - id: ruff-format - repo: https://github.com/dosisod/refurb - rev: v1.21.0 + rev: v2.0.0 hooks: - id: refurb - args: [--ignore, FURB126] -- repo: https://github.com/econchick/interrogate - rev: 1.5.0 - hooks: - - id: interrogate - args: [-v, --fail-under=40, src, tests] - repo: https://github.com/executablebooks/mdformat rev: 0.7.17 hooks: @@ -58,7 +44,7 @@ repos: hooks: - id: codespell - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.5.1' + rev: 'v1.9.0' hooks: - id: mypy additional_dependencies: [ @@ -74,7 +60,7 @@ repos: hooks: - id: check-manifest args: [--no-build-isolation] - additional_dependencies: [setuptools-scm, toml] + additional_dependencies: [setuptools-scm, wheel, toml] - repo: meta hooks: - id: check-hooks-apply diff --git a/README.md b/README.md index 9c29ed1..d38ab1d 100644 --- a/README.md +++ b/README.md @@ -190,8 +190,7 @@ Use the `serializer` keyword arguments of the `@pytask.mark.r` decorator with ```python @pytask.mark.r(script="script.r", serializer="yaml") -def task_example(): - ... +def task_example(): ... ``` And in your R script use @@ -214,8 +213,7 @@ import json @pytask.mark.r(script="script.r", serializer=json.dumps, suffix=".json") -def task_example(): - ... +def task_example(): ... ``` ### Configuration diff --git a/pyproject.toml b/pyproject.toml index 922ea07..060dd09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,11 +2,9 @@ requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.0"] build-backend = "setuptools.build_meta" - [tool.setuptools_scm] write_to = "src/pytask_r/_version.py" - [tool.mypy] files = ["src", "tests"] check_untyped_defs = true @@ -17,56 +15,41 @@ no_implicit_optional = true warn_redundant_casts = true warn_unused_ignores = true - [[tool.mypy.overrides]] module = "tests.*" disallow_untyped_defs = false ignore_errors = true - [tool.ruff] target-version = "py38" -select = ["ALL"] fix = true +unsafe-fixes = true + +[tool.ruff.lint] extend-ignore = [ - "TCH", - "TRY", - # Numpy docstyle - "D107", - "D203", - "D212", - "D213", - "D402", - "D413", - "D415", - "D416", - "D417", - # Others. - "D404", # Do not start module docstring with "This". - "RET504", # unnecessary variable assignment before return. "S101", # raise errors for asserts. "B905", # strict parameter for zip that was implemented in py310. - "I", # ignore isort "ANN101", # type annotating self "ANN102", # type annotating cls "FBT", # flake8-boolean-trap "EM", # flake8-errmsg "ANN401", # flake8-annotate typing.Any "PD", # pandas-vet - "COM812", # trailing comma missing, but black takes care of that + "COM812", # Comply with ruff-format + "ISC001", # Comply with ruff-format ] +select = ["ALL"] - -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "tests/*" = ["D", "ANN"] - -[tool.ruff.pydocstyle] +[tool.ruff.lint.pydocstyle] convention = "numpy" +[tool.ruff.lint.isort] +force-single-line = true [tool.pytest.ini_options] -# Do not add src since it messes with the loading of pytask-parallel as a plugin. testpaths = ["tests"] markers = [ "wip: Tests that are work-in-progress.", diff --git a/src/pytask_r/__init__.py b/src/pytask_r/__init__.py index 917f1d8..8c31e12 100644 --- a/src/pytask_r/__init__.py +++ b/src/pytask_r/__init__.py @@ -1,4 +1,5 @@ -"""This module contains the main namespace of pytask-r.""" +"""Contains the main namespace of pytask-r.""" + from __future__ import annotations try: diff --git a/src/pytask_r/collect.py b/src/pytask_r/collect.py index f79bdb7..d2a3e3b 100644 --- a/src/pytask_r/collect.py +++ b/src/pytask_r/collect.py @@ -1,4 +1,5 @@ """Collect tasks.""" + from __future__ import annotations import subprocess @@ -6,27 +7,31 @@ from pathlib import Path from typing import Any -from pytask import has_mark -from pytask import hookimpl -from pytask import is_task_function from pytask import Mark from pytask import NodeInfo -from pytask import parse_dependencies_from_task_function -from pytask import parse_products_from_task_function from pytask import PathNode from pytask import PTask from pytask import PythonNode -from pytask import remove_marks from pytask import Session from pytask import Task from pytask import TaskWithoutPath -from pytask_r.serialization import create_path_to_serialized +from pytask import has_mark +from pytask import hookimpl +from pytask import is_task_function +from pytask import parse_dependencies_from_task_function +from pytask import parse_products_from_task_function +from pytask import remove_marks + from pytask_r.serialization import SERIALIZERS +from pytask_r.serialization import create_path_to_serialized from pytask_r.shared import r def run_r_script( - _script: Path, _options: list[str], _serialized: Path, **kwargs: Any # noqa: ARG001 + _script: Path, + _options: list[str], + _serialized: Path, + **kwargs: Any, # noqa: ARG001 ) -> None: """Run an R script.""" cmd = ["Rscript", _script.as_posix(), *_options, str(_serialized)] @@ -49,10 +54,11 @@ def pytask_collect_task( # Parse @pytask.mark.r decorator. obj, marks = remove_marks(obj, "r") if len(marks) > 1: - raise ValueError( + msg = ( f"Task {name!r} has multiple @pytask.mark.r marks, but only one is " "allowed." ) + raise ValueError(msg) mark = _parse_r_mark( mark=marks[0], @@ -88,10 +94,11 @@ def pytask_collect_task( ) if not (isinstance(script_node, PathNode) and script_node.path.suffix == ".r"): - raise ValueError( + msg = ( "The 'script' keyword of the @pytask.mark.r decorator must point " f"to Julia file with the .r suffix, but it is {script_node}." ) + raise ValueError(msg) options_node = session.hook.pytask_collect_node( session=session, @@ -181,5 +188,4 @@ def _parse_r_mark( ) parsed_kwargs["suffix"] = suffix or proposed_suffix # type: ignore[assignment] - mark = Mark("r", (), parsed_kwargs) - return mark + return Mark("r", (), parsed_kwargs) diff --git a/src/pytask_r/config.py b/src/pytask_r/config.py index f107589..a9eca85 100644 --- a/src/pytask_r/config.py +++ b/src/pytask_r/config.py @@ -1,9 +1,11 @@ """Configure pytask.""" + from __future__ import annotations from typing import Any from pytask import hookimpl + from pytask_r.serialization import SERIALIZERS @@ -13,10 +15,11 @@ def pytask_parse_config(config: dict[str, Any]) -> None: config["markers"]["r"] = "Tasks which are executed with Rscript." config["r_serializer"] = config.get("r_serializer", "json") if config["r_serializer"] not in SERIALIZERS: - raise ValueError( + msg = ( f"'r_serializer' is {config['r_serializer']} and not one of " f"{list(SERIALIZERS)}." ) + raise ValueError(msg) config["r_suffix"] = config.get("r_suffix", "") config["r_options"] = _parse_value_or_whitespace_option(config.get("r_options")) @@ -27,4 +30,5 @@ def _parse_value_or_whitespace_option(value: Any) -> list[str] | None: return None if isinstance(value, list): return list(map(str, value)) - raise ValueError(f"'r_options' is {value} and not a list.") + msg = f"'r_options' is {value} and not a list." + raise ValueError(msg) diff --git a/src/pytask_r/execute.py b/src/pytask_r/execute.py index ce6d427..fabb9f9 100644 --- a/src/pytask_r/execute.py +++ b/src/pytask_r/execute.py @@ -1,15 +1,17 @@ """Execute tasks.""" + from __future__ import annotations import shutil from typing import Any -from pytask import get_marks -from pytask import hookimpl from pytask import PPathNode from pytask import PTask from pytask import PythonNode +from pytask import get_marks +from pytask import hookimpl from pytask.tree_util import tree_map + from pytask_r.serialization import serialize_keyword_arguments from pytask_r.shared import r @@ -20,9 +22,10 @@ def pytask_execute_task_setup(task: PTask) -> None: marks = get_marks(task, "r") if marks: if shutil.which("Rscript") is None: - raise RuntimeError( + msg = ( "Rscript is needed to run R scripts, but it is not found on your PATH." ) + raise RuntimeError(msg) assert len(marks) == 1 @@ -31,9 +34,9 @@ def pytask_execute_task_setup(task: PTask) -> None: assert suffix serialized_node: PythonNode = task.depends_on["_serialized"] # type: ignore[assignment] - serialized_node.value.parent.mkdir(parents=True, exist_ok=True) + serialized_node.value.parent.mkdir(parents=True, exist_ok=True) # type: ignore[union-attr] kwargs = collect_keyword_arguments(task) - serialize_keyword_arguments(serializer, serialized_node.value, kwargs) + serialize_keyword_arguments(serializer, serialized_node.value, kwargs) # type: ignore[arg-type] def collect_keyword_arguments(task: PTask) -> dict[str, Any]: diff --git a/src/pytask_r/plugin.py b/src/pytask_r/plugin.py index ec6efc8..7281a99 100644 --- a/src/pytask_r/plugin.py +++ b/src/pytask_r/plugin.py @@ -1,12 +1,18 @@ """Register hook specifications and implementations.""" + from __future__ import annotations -from pluggy import PluginManager +from typing import TYPE_CHECKING + from pytask import hookimpl + from pytask_r import collect from pytask_r import config from pytask_r import execute +if TYPE_CHECKING: + from pluggy import PluginManager + @hookimpl def pytask_add_hooks(pm: PluginManager) -> None: diff --git a/src/pytask_r/serialization.py b/src/pytask_r/serialization.py index 6b23842..e95bf35 100644 --- a/src/pytask_r/serialization.py +++ b/src/pytask_r/serialization.py @@ -1,4 +1,5 @@ -"""This module contains the code to serialize keyword arguments to the task.""" +"""Contains the code to serialize keyword arguments to the task.""" + from __future__ import annotations import json @@ -9,7 +10,6 @@ from pytask import PTask from pytask import PTaskWithPath - _HIDDEN_FOLDER = ".pytask/pytask-r" @@ -29,8 +29,7 @@ def create_path_to_serialized(task: PTask, suffix: str) -> Path: """Create path to serialized.""" parent = task.path.parent if isinstance(task, PTaskWithPath) else Path.cwd() file_name = create_file_name(task, suffix) - path = parent.joinpath(_HIDDEN_FOLDER, file_name).with_suffix(suffix) - return path + return parent.joinpath(_HIDDEN_FOLDER, file_name).with_suffix(suffix) def create_file_name(task: PTask, suffix: str) -> str: @@ -59,11 +58,10 @@ def serialize_keyword_arguments( if callable(serializer): serializer_func = serializer elif isinstance(serializer, str) and serializer in SERIALIZERS: - serializer_func = SERIALIZERS[serializer][ - "serializer" - ] # type: ignore[assignment] + serializer_func = SERIALIZERS[serializer]["serializer"] # type: ignore[assignment] else: - raise ValueError(f"Serializer {serializer!r} is not known.") + msg = f"Serializer {serializer!r} is not known." + raise ValueError(msg) serialized = serializer_func(kwargs) path_to_serialized.write_text(serialized) diff --git a/src/pytask_r/shared.py b/src/pytask_r/shared.py index 49d29f1..03d4618 100644 --- a/src/pytask_r/shared.py +++ b/src/pytask_r/shared.py @@ -1,12 +1,16 @@ -"""This module contains shared functions.""" +"""Contains shared functions.""" + from __future__ import annotations -from pathlib import Path +from typing import TYPE_CHECKING from typing import Any from typing import Callable from typing import Iterable from typing import Sequence +if TYPE_CHECKING: + from pathlib import Path + def r( *, diff --git a/tests/conftest.py b/tests/conftest.py index 64daf97..c8d16f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import pytest from click.testing import CliRunner - +from pytask import storage needs_rscript = pytest.mark.skipif( shutil.which("Rscript") is None, reason="R with Rscript needs to be installed." @@ -37,7 +37,7 @@ class SysPathsSnapshot: """A snapshot for sys.path.""" def __init__(self) -> None: - self.__saved = list(sys.path), list(sys.meta_path) + self.__saved = sys.path.copy(), sys.meta_path.copy() def restore(self) -> None: sys.path[:], sys.meta_path[:] = self.__saved @@ -48,7 +48,7 @@ class SysModulesSnapshot: def __init__(self, preserve: Callable[[str], bool] | None = None) -> None: self.__preserve = preserve - self.__saved = dict(sys.modules) + self.__saved = sys.modules.copy() def restore(self) -> None: if self.__preserve: @@ -87,6 +87,7 @@ def _restore_sys_path_and_module_after_test_execution(): class CustomCliRunner(CliRunner): def invoke(self, *args, **kwargs): """Restore sys.path and sys.modules after an invocation.""" + storage.create() with restore_sys_path_and_module_after_test_execution(): return super().invoke(*args, **kwargs) diff --git a/tests/test_execute.py b/tests/test_execute.py index dc67b85..3a51c74 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -4,11 +4,11 @@ from pathlib import Path import pytest -from pytask import build -from pytask import cli from pytask import ExitCode from pytask import Mark from pytask import Task +from pytask import build +from pytask import cli from pytask_r.execute import pytask_execute_task_setup from tests.conftest import needs_rscript diff --git a/tests/test_normal_execution_w_plugin.py b/tests/test_normal_execution_w_plugin.py index 78d67cc..3c8813f 100644 --- a/tests/test_normal_execution_w_plugin.py +++ b/tests/test_normal_execution_w_plugin.py @@ -1,11 +1,12 @@ """Contains tests which do not require the plugin and ensure normal execution.""" + from __future__ import annotations import textwrap import pytest -from pytask import cli from pytask import ExitCode +from pytask import cli @pytest.mark.end_to_end() diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 587c649..a6754a4 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -1,11 +1,12 @@ """Contains test which ensure that the plugin works with pytask-parallel.""" + from __future__ import annotations import textwrap import pytest -from pytask import cli from pytask import ExitCode +from pytask import cli from tests.conftest import needs_rscript from tests.conftest import parametrize_parse_code_serializer_suffix diff --git a/tests/test_parametrize.py b/tests/test_parametrize.py index 1fe4333..6e9cb94 100644 --- a/tests/test_parametrize.py +++ b/tests/test_parametrize.py @@ -3,8 +3,8 @@ import textwrap import pytest -from pytask import cli from pytask import ExitCode +from pytask import cli from tests.conftest import needs_rscript from tests.conftest import parametrize_parse_code_serializer_suffix