diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d4f55d..3f0eba4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: debug-statements - id: end-of-file-fixer - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.9.0 # Use the ref you want to point at + rev: v1.9.0 hooks: - id: python-check-blanket-noqa - id: python-check-mock-methods @@ -17,11 +17,6 @@ repos: - id: python-no-log-warn - id: python-use-type-annotations - id: text-unicode-replacement-char -- repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 - hooks: - - id: pyupgrade - args: [--py37-plus] - repo: https://github.com/asottile/reorder_python_imports rev: v3.9.0 hooks: @@ -40,27 +35,15 @@ repos: rev: 22.12.0 hooks: - id: black -- repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.215 hooks: - - id: flake8 - types: [python] - additional_dependencies: [ - flake8-alfred, - flake8-bugbear, - flake8-builtins, - flake8-comprehensions, - flake8-docstrings, - flake8-eradicate, - flake8-print, - flake8-pytest-style, - flake8-todo, - flake8-typing-imports, - flake8-unused-arguments, - pep8-naming, - pydocstyle, - Pygments, - ] + - id: ruff +- repo: https://github.com/dosisod/refurb + rev: v1.9.1 + hooks: + - id: refurb + args: [--ignore, FURB126] - repo: https://github.com/econchick/interrogate rev: 1.5.0 hooks: @@ -79,6 +62,21 @@ repos: rev: v2.2.2 hooks: - id: codespell +- repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v0.991' + hooks: + - id: mypy + args: [ + --no-strict-optional, + --ignore-missing-imports, + ] + additional_dependencies: [ + attrs>=21.3.0, + click, + types-PyYAML, + types-setuptools + ] + pass_filenames: false - repo: https://github.com/mgedmin/check-manifest rev: "0.49" hooks: diff --git a/CHANGES.md b/CHANGES.md index 759817b..d08f679 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask-r) and ## 0.3.0 - 2023-xx-xx - {pull}`33` deprecates INI configurations and aligns the plugin with pytask v0.3. +- {pull}`34` adds mypy, ruff and refurb. ## 0.2.0 - 2022-04-16 diff --git a/pyproject.toml b/pyproject.toml index afa5c36..551349f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,58 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "src/pytask_r/_version.py" + + +[tool.mypy] +files = ["src", "tests"] +check_untyped_defs = true +disallow_any_generics = true +disallow_incomplete_defs = true +disallow_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true + + +[[tool.mypy.overrides]] +module = "tests.*" +disallow_untyped_defs = false +ignore_errors = true + + +[tool.ruff] +target-version = "py37" +select = ["ALL"] +fix = true +extend-ignore = [ + # Numpy docstyle + "D107", + "D203", + "D212", + "D213", + "D402", + "D413", + "D415", + "D416", + "D417", + # Others. + "D404", # Do not start module docstring with "This". + "RET504", # unnecessary variable assignment before return. + "S101", # raise errors for asserts. + "B905", # strict parameter for zip that was implemented in py310. + "I", # ignore isort + "ANN101", # type annotating self + "ANN102", # type annotating cls + "FBT", # flake8-boolean-trap + "EM", # flake8-errmsg + "ANN401", # flake8-annotate typing.Any + "PD", # pandas-vet +] + + +[tool.ruff.per-file-ignores] +"tests/*" = ["D", "ANN"] + + +[tool.ruff.pydocstyle] +convention = "numpy" diff --git a/src/pytask_r/__init__.py b/src/pytask_r/__init__.py index f493641..917f1d8 100644 --- a/src/pytask_r/__init__.py +++ b/src/pytask_r/__init__.py @@ -1,3 +1,4 @@ +"""This module contains the main namespace of pytask-r.""" from __future__ import annotations try: diff --git a/src/pytask_r/collect.py b/src/pytask_r/collect.py index 97dea84..dad6ce6 100644 --- a/src/pytask_r/collect.py +++ b/src/pytask_r/collect.py @@ -5,6 +5,7 @@ import subprocess from pathlib import Path from types import FunctionType +from typing import Any from pytask import depends_on from pytask import has_mark @@ -13,6 +14,7 @@ from pytask import parse_nodes from pytask import produces from pytask import remove_marks +from pytask import Session from pytask import Task from pytask_r.serialization import SERIALIZERS from pytask_r.shared import r @@ -26,7 +28,9 @@ def run_r_script(script: Path, options: list[str], serialized: Path) -> None: @hookimpl -def pytask_collect_task(session, path, name, obj): +def pytask_collect_task( + session: Session, path: Path, name: str, obj: Any +) -> Task | None: """Perform some checks.""" __tracebackhide__ = True @@ -62,7 +66,7 @@ def pytask_collect_task(session, path, name, obj): task = Task( base_name=name, path=path, - function=_copy_func(run_r_script), + function=_copy_func(run_r_script), # type: ignore[arg-type] depends_on=dependencies, produces=products, markers=markers, @@ -85,28 +89,33 @@ def pytask_collect_task(session, path, name, obj): ) return task + return None -def _parse_r_mark(mark, default_options, default_serializer, default_suffix): +def _parse_r_mark( + mark: Mark, + default_options: list[str] | None, + default_serializer: str, + default_suffix: str, +) -> Mark: """Parse a Julia mark.""" script, options, serializer, suffix = r(**mark.kwargs) parsed_kwargs = {} - for arg_name, value, default in [ + for arg_name, value, default in ( ("script", script, None), ("options", options, default_options), ("serializer", serializer, default_serializer), - ]: - parsed_kwargs[arg_name] = value if value else default + ): + parsed_kwargs[arg_name] = value or default - if ( - isinstance(parsed_kwargs["serializer"], str) + proposed_suffix = ( + SERIALIZERS[parsed_kwargs["serializer"]]["suffix"] + if isinstance(parsed_kwargs["serializer"], str) and parsed_kwargs["serializer"] in SERIALIZERS - ): - proposed_suffix = SERIALIZERS[parsed_kwargs["serializer"]]["suffix"] - else: - proposed_suffix = default_suffix - parsed_kwargs["suffix"] = suffix if suffix else proposed_suffix + else default_suffix + ) + parsed_kwargs["suffix"] = suffix or proposed_suffix # type: ignore[assignment] mark = Mark("r", (), parsed_kwargs) return mark diff --git a/src/pytask_r/config.py b/src/pytask_r/config.py index a135fbc..15260db 100644 --- a/src/pytask_r/config.py +++ b/src/pytask_r/config.py @@ -1,12 +1,14 @@ """Configure pytask.""" from __future__ import annotations +from typing import Any + from pytask import hookimpl from pytask_r.serialization import SERIALIZERS @hookimpl -def pytask_parse_config(config): +def pytask_parse_config(config: dict[str, Any]) -> None: """Register the r marker.""" config["markers"]["r"] = "Tasks which are executed with Rscript." config["r_serializer"] = config.get("r_serializer", "json") @@ -23,7 +25,6 @@ def _parse_value_or_whitespace_option(value: str | None) -> None | str | list[st """Parse option which can hold a single value or values separated by new lines.""" if value is None: return None - elif isinstance(value, list): + if isinstance(value, list): return list(map(str, value)) - else: - raise ValueError(f"'r_options' is {value} and not a list.") + raise ValueError(f"'r_options' is {value} and not a list.") diff --git a/src/pytask_r/execute.py b/src/pytask_r/execute.py index 8d26982..b63f331 100644 --- a/src/pytask_r/execute.py +++ b/src/pytask_r/execute.py @@ -15,7 +15,7 @@ @hookimpl -def pytask_execute_task_setup(task): +def pytask_execute_task_setup(task: Task) -> None: """Perform some checks when a task marked with the r marker is executed.""" marks = get_marks(task, "r") if marks: diff --git a/src/pytask_r/parametrize.py b/src/pytask_r/parametrize.py index 0f5cefc..4933021 100644 --- a/src/pytask_r/parametrize.py +++ b/src/pytask_r/parametrize.py @@ -1,13 +1,14 @@ """Parametrize tasks.""" from __future__ import annotations +from typing import Any + import pytask from pytask import hookimpl @hookimpl -def pytask_parametrize_kwarg_to_marker(obj, kwargs): +def pytask_parametrize_kwarg_to_marker(obj: Any, kwargs: dict[Any, Any]) -> None: """Attach parametrized r arguments to the function with a marker.""" - if callable(obj): - if "r" in kwargs: - pytask.mark.r(**kwargs.pop("r"))(obj) + if callable(obj) and "r" in kwargs: + pytask.mark.r(**kwargs.pop("r"))(obj) diff --git a/src/pytask_r/plugin.py b/src/pytask_r/plugin.py index 5440ed8..4c10eec 100644 --- a/src/pytask_r/plugin.py +++ b/src/pytask_r/plugin.py @@ -1,6 +1,7 @@ """Register hook specifications and implementations.""" from __future__ import annotations +from pluggy import PluginManager from pytask import hookimpl from pytask_r import collect from pytask_r import config @@ -9,7 +10,7 @@ @hookimpl -def pytask_add_hooks(pm): +def pytask_add_hooks(pm: PluginManager) -> None: """Register hook implementations.""" pm.register(collect) pm.register(config) diff --git a/src/pytask_r/serialization.py b/src/pytask_r/serialization.py index a68b57f..195772d 100644 --- a/src/pytask_r/serialization.py +++ b/src/pytask_r/serialization.py @@ -50,14 +50,17 @@ def create_file_name(task: Task, suffix: str) -> str: def serialize_keyword_arguments( - serializer: str | Callable[dict[str, Any], str], + serializer: str | Callable[..., str], path_to_serialized: Path, kwargs: dict[str, Any], ) -> None: + """Serialize keyword arguments.""" if callable(serializer): serializer_func = serializer elif isinstance(serializer, str) and serializer in SERIALIZERS: - serializer_func = SERIALIZERS[serializer]["serializer"] + serializer_func = SERIALIZERS[serializer][ + "serializer" + ] # type: ignore[assignment] else: raise ValueError(f"Serializer {serializer!r} is not known.") diff --git a/src/pytask_r/shared.py b/src/pytask_r/shared.py index db9ccc8..6a9f6ca 100644 --- a/src/pytask_r/shared.py +++ b/src/pytask_r/shared.py @@ -1,6 +1,8 @@ +"""This module contains shared functions.""" from __future__ import annotations from pathlib import Path +from typing import Any from typing import Callable from typing import Iterable from typing import Sequence @@ -39,17 +41,9 @@ def r( return script, options, serializer, suffix -def _to_list(scalar_or_iter): +def _to_list(scalar_or_iter: Any) -> list[Any]: """Convert scalars and iterables to list. - Parameters - ---------- - scalar_or_iter : str or list - - Returns - ------- - list - Examples -------- >>> _to_list("a") diff --git a/tests/test_collect.py b/tests/test_collect.py index b9cd250..2ce9752 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -9,9 +9,9 @@ from pytask_r.serialization import SERIALIZERS -@pytest.mark.unit +@pytest.mark.unit() @pytest.mark.parametrize( - "args, kwargs, expectation, expected", + ("args", "kwargs", "expectation", "expected"), [ ( (), @@ -43,9 +43,16 @@ def test_r(args, kwargs, expectation, expected): assert result == expected -@pytest.mark.unit +@pytest.mark.unit() @pytest.mark.parametrize( - "mark, default_options, default_serializer, default_suffix, expectation, expected", + ( + "mark", + "default_options", + "default_serializer", + "default_suffix", + "expectation", + "expected", + ), [ ( Mark("r", (), {"script": "script.r"}), diff --git a/tests/test_config.py b/tests/test_config.py index 9624b64..e80d404 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4,7 +4,7 @@ from pytask import main -@pytest.mark.end_to_end +@pytest.mark.end_to_end() def test_marker_is_configured(tmp_path): session = main({"paths": tmp_path}) diff --git a/tests/test_execute.py b/tests/test_execute.py index 291b704..07ca95c 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -15,11 +15,11 @@ from tests.conftest import parametrize_parse_code_serializer_suffix -@pytest.mark.unit +@pytest.mark.unit() def test_pytask_execute_task_setup(monkeypatch): """Make sure that the task setup raises errors.""" # Act like r is installed since we do not test this. - monkeypatch.setattr("pytask_r.execute.shutil.which", lambda x: None) # noqa: U100 + monkeypatch.setattr("pytask_r.execute.shutil.which", lambda x: None) # noqa: ARG005 task = Task( base_name="task_example", @@ -33,7 +33,7 @@ def test_pytask_execute_task_setup(monkeypatch): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix @pytest.mark.parametrize("depends_on", ["'in_1.txt'", "['in_1.txt', 'in_2.txt']"]) def test_run_r_script( @@ -71,7 +71,7 @@ def task_run_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_run_r_script_w_task_decorator( runner, tmp_path, parse_config_code, serializer, suffix @@ -103,7 +103,7 @@ def run_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_raise_error_if_rscript_is_not_found( tmp_path, monkeypatch, parse_config_code, serializer, suffix @@ -127,7 +127,7 @@ def task_run_r_script(): tmp_path.joinpath("script.r").write_text(textwrap.dedent(r_script)) # Hide Rscript if available. - monkeypatch.setattr("pytask_r.execute.shutil.which", lambda x: None) # noqa: U100 + monkeypatch.setattr("pytask_r.execute.shutil.which", lambda x: None) # noqa: ARG005 session = main({"paths": tmp_path}) @@ -136,7 +136,7 @@ def task_run_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_run_r_script_w_saving_workspace( runner, tmp_path, parse_config_code, serializer, suffix @@ -172,7 +172,7 @@ def task_run_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_run_r_script_w_wrong_cmd_option( runner, tmp_path, parse_config_code, serializer, suffix @@ -208,7 +208,7 @@ def task_run_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() def test_run_r_script_w_custom_serializer(runner, tmp_path): task_source = """ import pytask @@ -239,7 +239,7 @@ def task_run_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() def test_run_r_script_fails_w_multiple_markers(runner, tmp_path): task_source = """ import pytask diff --git a/tests/test_normal_execution_w_plugin.py b/tests/test_normal_execution_w_plugin.py index 4a810eb..78d67cc 100644 --- a/tests/test_normal_execution_w_plugin.py +++ b/tests/test_normal_execution_w_plugin.py @@ -8,12 +8,12 @@ from pytask import ExitCode -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize( "dependencies", - [[], ["in.txt"], ["in_1.txt", "in_2.txt"]], + [(), ("in.txt",), ("in_1.txt", "in_2.txt")], ) -@pytest.mark.parametrize("products", [["out.txt"], ["out_1.txt", "out_2.txt"]]) +@pytest.mark.parametrize("products", [("out.txt",), ("out_1.txt", "out_2.txt")]) def test_execution_w_varying_dependencies_products( runner, tmp_path, dependencies, products ): diff --git a/tests/test_parallel.py b/tests/test_parallel.py index b7dbc6c..539057f 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -37,7 +37,7 @@ @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parallel_parametrization_over_source_files_w_parametrize( runner, tmp_path, parse_config_code, serializer, suffix @@ -95,7 +95,7 @@ def task_execute_r_script(): assert result.exit_code == ExitCode.OK duration_normal = time.time() - start - for name in ["1.rds", "2.rds"]: + for name in ("1.rds", "2.rds"): tmp_path.joinpath(name).unlink() start = time.time() @@ -107,7 +107,7 @@ def task_execute_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parallel_parametrization_over_source_files_w_loop( runner, tmp_path, parse_config_code, serializer, suffix @@ -153,7 +153,7 @@ def task_execute_r_script(): assert result.exit_code == ExitCode.OK duration_normal = time.time() - start - for name in ["1.rds", "2.rds"]: + for name in ("1.rds", "2.rds"): tmp_path.joinpath(name).unlink() start = time.time() @@ -165,7 +165,7 @@ def task_execute_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parallel_parametrization_over_source_file_w_parametrize( runner, tmp_path, parse_config_code, serializer, suffix @@ -201,7 +201,7 @@ def task_execute_r_script(): assert result.exit_code == ExitCode.OK duration_normal = time.time() - start - for name in ["0.rds", "1.rds"]: + for name in ("0.rds", "1.rds"): tmp_path.joinpath(name).unlink() start = time.time() @@ -213,7 +213,7 @@ def task_execute_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parallel_parametrization_over_source_file_w_loop( runner, tmp_path, parse_config_code, serializer, suffix @@ -252,7 +252,7 @@ def execute_r_script(): assert result.exit_code == ExitCode.OK duration_normal = time.time() - start - for name in ["0.rds", "1.rds"]: + for name in ("0.rds", "1.rds"): tmp_path.joinpath(name).unlink() start = time.time() diff --git a/tests/test_parametrize.py b/tests/test_parametrize.py index 075bc18..9697964 100644 --- a/tests/test_parametrize.py +++ b/tests/test_parametrize.py @@ -23,7 +23,7 @@ @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parametrized_execution_of_r_script_w_parametrize( runner, tmp_path, parse_config_code, serializer, suffix @@ -54,10 +54,10 @@ def task_run_r_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - for name, content in [ + for name, content in ( ("script_1.r", "Cities breaking down on a camel's back"), ("script_2.r", "They just have to go 'cause they don't know whack"), - ]: + ): r_script = f""" {parse_config_code} file_descr <- file(config$produces) @@ -74,7 +74,7 @@ def task_run_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parametrized_execution_of_r_script_w_loop( runner, tmp_path, parse_config_code, serializer, suffix @@ -96,10 +96,10 @@ def task_run_r_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - for name, content in [ + for name, content in ( ("script_1.r", "Cities breaking down on a camel's back"), ("script_2.r", "They just have to go 'cause they don't know whack"), - ]: + ): r_script = f""" {parse_config_code} file_descr <- file(config$produces) @@ -116,7 +116,7 @@ def task_run_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parametrize_r_options_and_product_paths_w_parametrize( runner, tmp_path, parse_config_code, serializer, suffix @@ -149,7 +149,7 @@ def task_execute_r_script(): @needs_rscript -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parametrize_r_options_and_product_paths_w_loop( runner, tmp_path, parse_config_code, serializer, suffix