diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4d350f6..5669061 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: check-added-large-files args: ['--maxkb=100'] @@ -24,50 +24,33 @@ repos: - id: python-no-log-warn - id: python-use-type-annotations - id: text-unicode-replacement-char -- repo: https://github.com/asottile/pyupgrade - rev: v3.1.0 - hooks: - - id: pyupgrade - args: [--py37-plus] - repo: https://github.com/asottile/reorder_python_imports - rev: v3.8.5 + rev: v3.9.0 hooks: - id: reorder-python-imports args: [--py37-plus, --add-import, 'from __future__ import annotations'] - repo: https://github.com/asottile/setup-cfg-fmt - rev: v2.1.0 + rev: v2.2.0 hooks: - id: setup-cfg-fmt - repo: https://github.com/PyCQA/docformatter - rev: v1.5.0 + rev: v1.5.1 hooks: - id: docformatter args: [--in-place, --wrap-summaries, "88", --wrap-descriptions, "88", --blank] - repo: https://github.com/psf/black - rev: 22.10.0 + rev: 22.12.0 hooks: - id: black -- repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.215 hooks: - - id: flake8 - types: [python] - additional_dependencies: [ - flake8-alfred, - flake8-bugbear, - flake8-builtins, - flake8-comprehensions, - flake8-docstrings, - flake8-eradicate, - flake8-print, - flake8-pytest-style, - flake8-todo, - flake8-typing-imports, - flake8-unused-arguments, - pep8-naming, - pydocstyle, - Pygments, - ] + - id: ruff +- repo: https://github.com/dosisod/refurb + rev: v1.9.1 + hooks: + - id: refurb + args: [--ignore, FURB126] - repo: https://github.com/executablebooks/mdformat rev: 0.7.16 hooks: @@ -82,12 +65,27 @@ repos: hooks: - id: interrogate args: [-v, --fail-under=40, src] +- repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v0.991' + hooks: + - id: mypy + args: [ + --no-strict-optional, + --ignore-missing-imports, + ] + additional_dependencies: [ + attrs>=21.3.0, + click, + types-PyYAML, + types-setuptools + ] + pass_filenames: false - repo: https://github.com/codespell-project/codespell rev: v2.2.2 hooks: - id: codespell - repo: https://github.com/mgedmin/check-manifest - rev: "0.48" + rev: "0.49" hooks: - id: check-manifest args: [--no-build-isolation] diff --git a/CHANGES.md b/CHANGES.md index a164ed6..73be651 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask-julia) and ## 0.3.0 - 2023-xx-xx +- {pull}`16` adds mypy, refurb, and ruff. - {pull}`18` deprecates INI configurations and aligns the package with pytask v0.3. ## 0.2.1 - 2022-04-16 diff --git a/MANIFEST.in b/MANIFEST.in index a5bb538..81a6ce6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,3 +10,5 @@ exclude *.yml include README.md include LICENSE include pyproject.toml + +recursive-include src py.typed diff --git a/pyproject.toml b/pyproject.toml index 4d35690..1e5a558 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,58 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "src/pytask_julia/_version.py" + + +[tool.mypy] +files = ["src", "tests"] +check_untyped_defs = true +disallow_any_generics = true +disallow_incomplete_defs = true +disallow_untyped_defs = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true + + +[[tool.mypy.overrides]] +module = "tests.*" +disallow_untyped_defs = false +ignore_errors = true + + +[tool.ruff] +target-version = "py37" +select = ["ALL"] +fix = true +extend-ignore = [ + # Numpy docstyle + "D107", + "D203", + "D212", + "D213", + "D402", + "D413", + "D415", + "D416", + "D417", + # Others. + "D404", # Do not start module docstring with "This". + "RET504", # unnecessary variable assignment before return. + "S101", # raise errors for asserts. + "B905", # strict parameter for zip that was implemented in py310. + "I", # ignore isort + "ANN101", # type annotating self + "ANN102", # type annotating cls + "FBT", # flake8-boolean-trap + "EM", # flake8-errmsg + "ANN401", # flake8-annotate typing.Any + "PD", # pandas-vet +] + + +[tool.ruff.per-file-ignores] +"tests/*" = ["D", "ANN"] + + +[tool.ruff.pydocstyle] +convention = "numpy" diff --git a/src/pytask_julia/__init__.py b/src/pytask_julia/__init__.py index dcc8e73..2a4567b 100644 --- a/src/pytask_julia/__init__.py +++ b/src/pytask_julia/__init__.py @@ -1,3 +1,4 @@ +"""This module contains the main namespace.""" from __future__ import annotations try: diff --git a/src/pytask_julia/collect.py b/src/pytask_julia/collect.py index aaee6bd..005b26a 100644 --- a/src/pytask_julia/collect.py +++ b/src/pytask_julia/collect.py @@ -5,6 +5,8 @@ import subprocess import types from pathlib import Path +from typing import Any +from typing import Callable from pytask import depends_on from pytask import has_mark @@ -13,6 +15,7 @@ from pytask import parse_nodes from pytask import produces from pytask import remove_marks +from pytask import Session from pytask import Task from pytask_julia.serialization import SERIALIZERS from pytask_julia.shared import julia @@ -33,12 +36,15 @@ def run_jl_script( @hookimpl -def pytask_collect_task(session, path, name, obj): +def pytask_collect_task( + session: Session, path: Path, name: str, obj: Any +) -> Task | None: """Collect a task which is a function. There is some discussion on how to detect functions in this `thread - `_. :class:`types.FunctionType` does not - detect built-ins which is not possible anyway. + + `_. :class:`types.FunctionType` + does notdetect built-ins which is not possible anyway. """ __tracebackhide__ = True @@ -76,7 +82,7 @@ def pytask_collect_task(session, path, name, obj): task = Task( base_name=name, path=path, - function=_copy_func(run_jl_script), + function=_copy_func(run_jl_script), # type: ignore[arg-type] depends_on=dependencies, produces=products, markers=markers, @@ -104,30 +110,34 @@ def pytask_collect_task(session, path, name, obj): ) return task + return None def _parse_julia_mark( - mark, default_options, default_serializer, default_suffix, default_project -): + mark: Mark, + default_options: list[str] | None, + default_serializer: Callable[..., str] | str | None, + default_suffix: str | None, + default_project: str | None, +) -> Mark: """Parse a Julia mark.""" script, options, serializer, suffix, project = julia(**mark.kwargs) parsed_kwargs = {} - for arg_name, value, default in [ + for arg_name, value, default in ( ("script", script, None), ("options", options, default_options), ("serializer", serializer, default_serializer), - ]: - parsed_kwargs[arg_name] = value if value else default + ): + parsed_kwargs[arg_name] = value or default - if ( - isinstance(parsed_kwargs["serializer"], str) + proposed_suffix = ( + SERIALIZERS[parsed_kwargs["serializer"]]["suffix"] + if isinstance(parsed_kwargs["serializer"], str) and parsed_kwargs["serializer"] in SERIALIZERS - ): - proposed_suffix = SERIALIZERS[parsed_kwargs["serializer"]]["suffix"] - else: - proposed_suffix = default_suffix - parsed_kwargs["suffix"] = suffix if suffix else proposed_suffix + else default_suffix + ) + parsed_kwargs["suffix"] = suffix or proposed_suffix # type: ignore[assignment] if isinstance(project, (str, Path)): parsed_kwargs["project"] = project diff --git a/src/pytask_julia/config.py b/src/pytask_julia/config.py index 5b2e57d..84b877d 100644 --- a/src/pytask_julia/config.py +++ b/src/pytask_julia/config.py @@ -9,7 +9,7 @@ @hookimpl -def pytask_parse_config(config): +def pytask_parse_config(config: dict[str, Any]) -> None: """Register the julia marker.""" config["markers"]["julia"] = "Tasks which are executed with Julia." config["julia_serializer"] = config.get("julia_serializer", "json") @@ -35,5 +35,4 @@ def _parse_value_or_whitespace_option(value: Any) -> None | list[str]: return None if isinstance(value, list): return list(map(str, value)) - else: - raise ValueError(f"'julia_options' is {value} and not a list.") + raise ValueError(f"'julia_options' is {value} and not a list.") diff --git a/src/pytask_julia/execute.py b/src/pytask_julia/execute.py index 64c4a19..d3680a7 100644 --- a/src/pytask_julia/execute.py +++ b/src/pytask_julia/execute.py @@ -15,7 +15,7 @@ @hookimpl -def pytask_execute_task_setup(task): +def pytask_execute_task_setup(task: Task) -> None: """Check whether environment allows executing Julia files.""" marks = get_marks(task, "julia") if marks: diff --git a/src/pytask_julia/parametrize.py b/src/pytask_julia/parametrize.py index 6e1d1d0..06193c7 100644 --- a/src/pytask_julia/parametrize.py +++ b/src/pytask_julia/parametrize.py @@ -1,13 +1,14 @@ """Parametrize tasks.""" from __future__ import annotations +from typing import Any + import pytask from pytask import hookimpl @hookimpl -def pytask_parametrize_kwarg_to_marker(obj, kwargs): +def pytask_parametrize_kwarg_to_marker(obj: Any, kwargs: dict[str, Any]) -> None: """Attach parametrized Julia arguments to the function with a marker.""" - if callable(obj): - if "julia" in kwargs: - pytask.mark.julia(**kwargs.pop("julia"))(obj) + if callable(obj) and "julia" in kwargs: + pytask.mark.julia(**kwargs.pop("julia"))(obj) diff --git a/src/pytask_julia/plugin.py b/src/pytask_julia/plugin.py index 023f84b..0194eaa 100644 --- a/src/pytask_julia/plugin.py +++ b/src/pytask_julia/plugin.py @@ -1,6 +1,7 @@ """Register hook specifications and implementations.""" from __future__ import annotations +from pluggy import PluginManager from pytask import hookimpl from pytask_julia import collect from pytask_julia import config @@ -9,7 +10,7 @@ @hookimpl -def pytask_add_hooks(pm): +def pytask_add_hooks(pm: PluginManager) -> None: """Register hook implementations.""" pm.register(collect) pm.register(config) diff --git a/src/pytask_julia/py.typed b/src/pytask_julia/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/pytask_julia/serialization.py b/src/pytask_julia/serialization.py index 57c923d..7199e1c 100644 --- a/src/pytask_julia/serialization.py +++ b/src/pytask_julia/serialization.py @@ -51,14 +51,17 @@ def create_file_name(task: Task, suffix: str) -> str: def serialize_keyword_arguments( - serializer: str | Callable[dict[str, Any], str], + serializer: str | Callable[..., str], path_to_serialized: Path, kwargs: dict[str, Any], ) -> None: + """Serialize keyword arguments.""" if callable(serializer): serializer_func = serializer elif isinstance(serializer, str) and serializer in SERIALIZERS: - serializer_func = SERIALIZERS[serializer]["serializer"] + serializer_func = SERIALIZERS[serializer][ + "serializer" + ] # type: ignore[assignment] else: # pragma: no cover raise ValueError(f"Serializer {serializer!r} is not known.") diff --git a/src/pytask_julia/shared.py b/src/pytask_julia/shared.py index 95a15b9..f6d8223 100644 --- a/src/pytask_julia/shared.py +++ b/src/pytask_julia/shared.py @@ -1,6 +1,8 @@ +"""This module contains shared functions.""" from __future__ import annotations from pathlib import Path +from typing import Any from typing import Callable from typing import Iterable from typing import Sequence @@ -9,7 +11,7 @@ def julia( script: str | Path, options: str | Iterable[str] | None = None, - serializer: str | Callable[..., str] | str | None = None, + serializer: Callable[..., str] | str | None = None, suffix: str | None = None, project: str | Path = None, ) -> tuple[ @@ -43,7 +45,7 @@ def julia( return script, options, serializer, suffix, project -def _to_list(scalar_or_iter): +def _to_list(scalar_or_iter: Any) -> list[Any]: """Convert scalars and iterables to list. Parameters diff --git a/tests/test_collect.py b/tests/test_collect.py index 953c1bc..2388b76 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -11,10 +11,17 @@ from tests.conftest import ROOT -@pytest.mark.unit +@pytest.mark.unit() @pytest.mark.parametrize( - "mark, default_options, default_serializer, default_suffix, default_project, " - "expectation, expected", + ( + "mark", + "default_options", + "default_serializer", + "default_suffix", + "default_project", + "expectation", + "expected", + ), [ ( Mark("julia", (), {"script": "script.jl"}), @@ -80,9 +87,9 @@ def test_parse_julia_mark( assert out == expected -@pytest.mark.unit +@pytest.mark.unit() @pytest.mark.parametrize( - "project, root, expected", + ("project", "root", "expected"), [ (None, ROOT, []), ], diff --git a/tests/test_config.py b/tests/test_config.py index 8f57086..cb0dc03 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4,7 +4,7 @@ from pytask import main -@pytest.mark.end_to_end +@pytest.mark.end_to_end() def test_marker_is_configured(tmp_path): session = main({"paths": tmp_path}) diff --git a/tests/test_execute.py b/tests/test_execute.py index feb9ead..589db09 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -18,12 +18,12 @@ from tests.conftest import ROOT -@pytest.mark.unit +@pytest.mark.unit() def test_pytask_execute_task_setup_missing_julia(monkeypatch): """Make sure that the task setup raises errors.""" # Act like julia is installed since we do not test this. monkeypatch.setattr( - "pytask_julia.execute.shutil.which", lambda x: None # noqa: U100 + "pytask_julia.execute.shutil.which", lambda x: None # noqa: ARG005 ) task = Task( base_name="example", path=Path(), function=None, markers=[Mark("julia", (), {})] @@ -33,7 +33,7 @@ def test_pytask_execute_task_setup_missing_julia(monkeypatch): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix @pytest.mark.parametrize("depends_on", ["'in_1.txt'", "['in_1.txt', 'in_2.txt']"]) def test_run_jl_script( @@ -80,7 +80,7 @@ def task_run_jl_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_run_jl_script_w_task_decorator( runner, tmp_path, parse_config_code, serializer, suffix @@ -115,7 +115,7 @@ def run_jl_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_raise_error_if_julia_is_not_found( tmp_path, monkeypatch, parse_config_code, serializer, suffix @@ -146,7 +146,7 @@ def task_run_jl_script(): # Hide julia if available. monkeypatch.setattr( - "pytask_julia.execute.shutil.which", lambda x: None # noqa: U100 + "pytask_julia.execute.shutil.which", lambda x: None # noqa: ARG005 ) session = main({"paths": tmp_path}) @@ -156,7 +156,7 @@ def task_run_jl_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_run_jl_script_w_wrong_cmd_option( runner, tmp_path, parse_config_code, serializer, suffix @@ -191,7 +191,7 @@ def task_run_jl_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize("n_threads", [2, 3]) @parametrize_parse_code_serializer_suffix def test_check_passing_cmd_line_options( @@ -227,14 +227,14 @@ def task_run_jl_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.xfail( condition=sys.platform == "win32" and os.environ.get("CI") == "true", reason="Test folder and repo are on different drives causing relpath to fail.", ) @parametrize_parse_code_serializer_suffix @pytest.mark.parametrize( - "config_path, value", + ("config_path", "value"), [ ("pytask.ini", "[pytask]\njulia_project={}"), ("pyproject.toml", "[tool.pytask.ini_options]\njulia_project='{}'"), @@ -267,10 +267,11 @@ def task_run_jl_script(): """ tmp_path.joinpath("script.jl").write_text(textwrap.dedent(julia_script)) - if isinstance(path, Path): - path_in_config = path.as_posix() - else: - path_in_config = Path(os.path.relpath(ROOT, tmp_path)).as_posix() + path_in_config = ( + path.as_posix() + if isinstance(path, Path) + else Path(os.path.relpath(ROOT, tmp_path)).as_posix() + ) tmp_path.joinpath(config_path).write_text(value.format(path_in_config)) result = runner.invoke(cli, [tmp_path.as_posix()]) @@ -283,7 +284,7 @@ def task_run_jl_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.xfail( condition=sys.platform == "win32" and os.environ.get("CI") == "true", reason="Test folder and repo are on different drives causing relpath to fail.", @@ -328,7 +329,7 @@ def task_run_jl_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() def test_run_jl_script_w_custom_serializer(runner, tmp_path): task_source = f""" import pytask @@ -362,7 +363,7 @@ def task_run_jl_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() def test_run_jl_script_fails_w_multiple_markers(runner, tmp_path): task_source = """ import pytask diff --git a/tests/test_normal_execution_w_plugin.py b/tests/test_normal_execution_w_plugin.py index 4a810eb..78d67cc 100644 --- a/tests/test_normal_execution_w_plugin.py +++ b/tests/test_normal_execution_w_plugin.py @@ -8,12 +8,12 @@ from pytask import ExitCode -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize( "dependencies", - [[], ["in.txt"], ["in_1.txt", "in_2.txt"]], + [(), ("in.txt",), ("in_1.txt", "in_2.txt")], ) -@pytest.mark.parametrize("products", [["out.txt"], ["out_1.txt", "out_2.txt"]]) +@pytest.mark.parametrize("products", [("out.txt",), ("out_1.txt", "out_2.txt")]) def test_execution_w_varying_dependencies_products( runner, tmp_path, dependencies, products ): diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 8221a78..ccdf5e3 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -33,7 +33,7 @@ @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parallel_parametrization_over_source_files_w_parametrize( runner, tmp_path, parse_config_code, serializer, suffix @@ -92,7 +92,7 @@ def task_execute_julia(): assert result.exit_code == ExitCode.OK duration_normal = time.time() - start - for name in ["1.csv", "2.csv"]: + for name in ("1.csv", "2.csv"): tmp_path.joinpath(name).unlink() start = time.time() @@ -104,7 +104,7 @@ def task_execute_julia(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parallel_parametrization_over_source_files_w_loop( runner, tmp_path, parse_config_code, serializer, suffix @@ -151,7 +151,7 @@ def task_execute_julia(): assert result.exit_code == ExitCode.OK duration_normal = time.time() - start - for name in ["1.csv", "2.csv"]: + for name in ("1.csv", "2.csv"): tmp_path.joinpath(name).unlink() start = time.time() @@ -163,7 +163,7 @@ def task_execute_julia(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parallel_parametrization_over_source_file_w_parametrize( runner, tmp_path, parse_config_code, serializer, suffix @@ -200,7 +200,7 @@ def task_execute_julia_script(): assert result.exit_code == ExitCode.OK duration_normal = time.time() - start - for name in ["0.csv", "1.csv"]: + for name in ("0.csv", "1.csv"): tmp_path.joinpath(name).unlink() start = time.time() @@ -212,7 +212,7 @@ def task_execute_julia_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parallel_parametrization_over_source_file_w_loop( runner, tmp_path, parse_config_code, serializer, suffix @@ -252,7 +252,7 @@ def task_execute_julia_script(): assert result.exit_code == ExitCode.OK duration_normal = time.time() - start - for name in ["0.csv", "1.csv"]: + for name in ("0.csv", "1.csv"): tmp_path.joinpath(name).unlink() start = time.time() diff --git a/tests/test_parametrize.py b/tests/test_parametrize.py index f8b510f..3260e1b 100644 --- a/tests/test_parametrize.py +++ b/tests/test_parametrize.py @@ -19,7 +19,7 @@ @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parametrized_execution_of_jl_script_w_parametrize( runner, tmp_path, parse_config_code, serializer, suffix @@ -54,7 +54,7 @@ def task_run_jl_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - for name in ["script_1.jl", "script_2.jl"]: + for name in ("script_1.jl", "script_2.jl"): julia_script = f""" {parse_config_code} write(config["produces"], config["content"]) @@ -66,7 +66,7 @@ def task_run_jl_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parametrized_execution_of_jl_script_w_loop( runner, tmp_path, parse_config_code, serializer, suffix @@ -92,7 +92,7 @@ def task_run_jl_script(): """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) - for name in ["script_1.jl", "script_2.jl"]: + for name in ("script_1.jl", "script_2.jl"): julia_script = f""" {parse_config_code} write(config["produces"], config["content"]) @@ -104,7 +104,7 @@ def task_run_jl_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parametrize_jl_options_and_product_paths_w_parametrize( runner, tmp_path, parse_config_code, serializer, suffix @@ -151,7 +151,7 @@ def task_run_jl_script(): @needs_julia -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @parametrize_parse_code_serializer_suffix def test_parametrize_jl_options_and_product_paths_w_loop( runner, tmp_path, parse_config_code, serializer, suffix diff --git a/tests/test_shared.py b/tests/test_shared.py index 2525e08..39a0206 100644 --- a/tests/test_shared.py +++ b/tests/test_shared.py @@ -6,9 +6,9 @@ from pytask_julia.shared import julia -@pytest.mark.unit +@pytest.mark.unit() @pytest.mark.parametrize( - "args, kwargs, expectation, expected", + ("args", "kwargs", "expectation", "expected"), [ ( (),