diff --git a/MANIFEST.in b/MANIFEST.in index 44e6bda9c..84eeefdb0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ include CITATION include LICENSE +recursive-include src *.pyi recursive-include src py.typed exclude .coveragerc diff --git a/pyproject.toml b/pyproject.toml index dc6b12a1e..d494d5b14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,3 +92,4 @@ markers = [ "end_to_end: Flag for tests that cover the whole program.", ] norecursedirs = [".idea", ".tox"] +filterwarnings = ["ignore:'@pytask.mark.*. is deprecated:DeprecationWarning"] diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 19c4d924e..b333d2c66 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -3,6 +3,7 @@ import itertools import uuid +import warnings from pathlib import Path from typing import Any from typing import Callable @@ -21,6 +22,7 @@ from _pytask.nodes import PythonNode from _pytask.shared import find_duplicates from _pytask.task_utils import parse_keyword_arguments_from_signature_defaults +from _pytask.tree_util import PyTree from _pytask.tree_util import tree_leaves from _pytask.tree_util import tree_map from _pytask.tree_util import tree_map_with_path @@ -42,9 +44,7 @@ ] -def depends_on( - objects: Any | Iterable[Any] | dict[Any, Any] -) -> Any | Iterable[Any] | dict[Any, Any]: +def depends_on(objects: PyTree[Any]) -> PyTree[Any]: """Specify dependencies for a task. Parameters @@ -58,9 +58,7 @@ def depends_on( return objects -def produces( - objects: Any | Iterable[Any] | dict[Any, Any] -) -> Any | Iterable[Any] | dict[Any, Any]: +def produces(objects: PyTree[Any]) -> PyTree[Any]: """Specify products of a task. Parameters @@ -342,6 +340,18 @@ def _find_args_with_node_annotation(func: Callable[..., Any]) -> dict[str, MetaN Read more about products in the documentation: https://tinyurl.com/yrezszr4. """ +_WARNING_PRODUCES_AS_KWARG = """Using 'produces' as an argument name to specify \ +products is deprecated and won't be available in pytask v0.5. Instead, use the product \ +annotation, described in this tutorial: https://tinyurl.com/yrezszr4. + + from typing_extensions import Annotated + from pytask import Product + + def task_example(produces: Annotated[..., Product]): + ... + +""" + def parse_products_from_task_function( session: Session, path: Path, name: str, obj: Any @@ -369,8 +379,14 @@ def parse_products_from_task_function( signature_defaults = parse_keyword_arguments_from_signature_defaults(obj) kwargs = {**signature_defaults, **task_kwargs} + parameters_with_product_annot = _find_args_with_product_annotation(obj) + # Parse products from task decorated with @task and that uses produces. if "produces" in kwargs: + if "produces" not in parameters_with_product_annot: + warnings.warn( + _WARNING_PRODUCES_AS_KWARG, category=FutureWarning, stacklevel=1 + ) has_produces_argument = True collected_products = tree_map_with_path( lambda p, x: _collect_product( @@ -384,7 +400,6 @@ def parse_products_from_task_function( ) out = {"produces": collected_products} - parameters_with_product_annot = _find_args_with_product_annotation(obj) if parameters_with_product_annot: has_annotation = True for parameter_name in parameters_with_product_annot: @@ -436,6 +451,11 @@ def _find_args_with_product_annotation(func: Callable[..., Any]) -> list[str]: """ +_WARNING_STRING_DEPRECATED = """Using strings to specify a {kind} is deprecated. Pass \ +a 'pathlib.Path' instead with 'Path("{node}")'. +""" + + def _collect_decorator_nodes( session: Session, path: Path, name: str, node_info: NodeInfo ) -> dict[str, MetaNode]: @@ -448,6 +468,7 @@ def _collect_decorator_nodes( """ node = node_info.value + kind = {"depends_on": "dependency", "produces": "product"}.get(node_info.arg_name) if not isinstance(node, (str, Path)): raise NodeNotCollectedError( @@ -455,6 +476,11 @@ def _collect_decorator_nodes( ) if isinstance(node, str): + warnings.warn( + _WARNING_STRING_DEPRECATED.format(kind=kind, node=node), + category=FutureWarning, + stacklevel=1, + ) node = Path(node) node_info = node_info._replace(value=node) @@ -462,9 +488,6 @@ def _collect_decorator_nodes( session=session, path=path, node_info=node_info ) if collected_node is None: - kind = {"depends_on": "dependency", "produces": "product"}.get( - node_info.arg_name - ) raise NodeNotCollectedError( f"{node!r} cannot be parsed as a {kind} for task {name!r} in {path!r}." ) @@ -525,10 +548,9 @@ def _collect_product( # The parameter defaults only support Path objects. if not isinstance(node, Path) and not is_string_allowed: raise ValueError( - "If you use 'produces' as a function argument of a task and pass values as " - "function defaults, it can only accept values of type 'pathlib.Path' or " - "the same value nested in tuples, lists, and dictionaries. Here, " - f"{node!r} has type {type(node)}." + "If you declare products with 'Annotated[..., Product]', only values of " + "type 'pathlib.Path' optionally nested in tuples, lists, and " + f"dictionaries are allowed. Here, {node!r} has type {type(node)}." ) if isinstance(node, str): diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py index 25db802b4..83a603291 100644 --- a/src/_pytask/mark/__init__.py +++ b/src/_pytask/mark/__init__.py @@ -39,6 +39,8 @@ "MarkDecorator", "MarkGenerator", "ParseError", + "select_by_keyword", + "select_by_mark", ] diff --git a/src/_pytask/mark/__init__.pyi b/src/_pytask/mark/__init__.pyi new file mode 100644 index 000000000..eca5e6e3b --- /dev/null +++ b/src/_pytask/mark/__init__.pyi @@ -0,0 +1,41 @@ +from pathlib import Path +from typing_extensions import deprecated +from _pytask.mark.expression import Expression +from _pytask.mark.expression import ParseError +from _pytask.mark.structures import Mark +from _pytask.mark.structures import MarkDecorator +from _pytask.mark.structures import MarkGenerator +from _pytask.tree_util import PyTree + +from _pytask.session import Session +import networkx as nx + +def select_by_keyword(session: Session, dag: nx.DiGraph) -> set[str]: ... +def select_by_mark(session: Session, dag: nx.DiGraph) -> set[str]: ... + +class MARK_GEN: # noqa: N801 + @deprecated( + "'@pytask.mark.produces' is deprecated starting pytask v0.4.0 and will be removed in v0.5.0. To upgrade your project to the new syntax, read the tutorial on product and dependencies: https://tinyurl.com/yrezszr4.", # noqa: E501 + category=DeprecationWarning, + stacklevel=1, + ) + @staticmethod + def produces(objects: PyTree[str | Path]) -> None: ... + @deprecated( + "'@pytask.mark.depends_on' is deprecated starting pytask v0.4.0 and will be removed in v0.5.0. To upgrade your project to the new syntax, read the tutorial on product and dependencies: https://tinyurl.com/yrezszr4.", # noqa: E501 + category=DeprecationWarning, + stacklevel=1, + ) + @staticmethod + def depends_on(objects: PyTree[str | Path]) -> None: ... + +__all__ = [ + "Expression", + "MARK_GEN", + "Mark", + "MarkDecorator", + "MarkGenerator", + "ParseError", + "select_by_keyword", + "select_by_mark", +] diff --git a/src/_pytask/mark/structures.py b/src/_pytask/mark/structures.py index 4dae51e67..62109f495 100644 --- a/src/_pytask/mark/structures.py +++ b/src/_pytask/mark/structures.py @@ -167,6 +167,12 @@ def store_mark(obj: Callable[..., Any], mark: Mark) -> None: ) +_DEPRECATION_DECORATOR = """'@pytask.mark.{}' is deprecated starting pytask \ +v0.4.0 and will be removed in v0.5.0. To upgrade your project to the new syntax, read \ +the tutorial on product and dependencies: https://tinyurl.com/yrezszr4. +""" + + class MarkGenerator: """Factory for :class:`MarkDecorator` objects. @@ -191,6 +197,13 @@ def __getattr__(self, name: str) -> MarkDecorator | Any: if name[0] == "_": raise AttributeError("Marker name must NOT start with underscore") + if name in ("depends_on", "produces"): + warnings.warn( + _DEPRECATION_DECORATOR.format(name), + category=DeprecationWarning, + stacklevel=1, + ) + # If the name is not in the set of known marks after updating, # then it really is time to issue a warning or an error. if self.config is not None and name not in self.config["markers"]: diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index faeb1e03a..e0b29d3bf 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -47,7 +47,7 @@ class _ExportFormats(enum.Enum): CSV = "csv" -class Runtime(BaseTable): # type: ignore[valid-type, misc] +class Runtime(BaseTable): """Record of runtimes of tasks.""" __tablename__ = "runtime" diff --git a/tests/test_collect.py b/tests/test_collect.py index bb90f5d2b..db951b15e 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -312,7 +312,7 @@ def task_my_task(): @pytest.mark.end_to_end() -def test_collect_string_product_with_task_decorator(tmp_path): +def test_collect_string_product_with_task_decorator(runner, tmp_path): source = """ import pytask @@ -321,25 +321,38 @@ def task_write_text(produces="out.txt"): produces.touch() """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) - assert session.exit_code == ExitCode.OK + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK assert tmp_path.joinpath("out.txt").exists() @pytest.mark.end_to_end() -def test_collect_string_product_as_function_default(tmp_path): +def test_collect_string_product_as_function_default(runner, tmp_path): source = """ - import pytask - def task_write_text(produces="out.txt"): produces.touch() """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) - assert session.exit_code == ExitCode.OK + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK assert tmp_path.joinpath("out.txt").exists() +@pytest.mark.end_to_end() +def test_collect_string_product_raises_error_with_annotation(runner, tmp_path): + source = """ + from pytask import Product + from typing_extensions import Annotated + + def task_write_text(out: Annotated[str, Product] = "out.txt") -> None: + out.touch() + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.COLLECTION_FAILED + assert "If you declare products with 'Annotated[..., Product]'" in result.output + + @pytest.mark.end_to_end() def test_product_cannot_mix_different_product_types(tmp_path): source = """ @@ -389,3 +402,23 @@ def task_example( report = session.collection_reports[0] assert report.outcome == CollectionOutcome.FAIL assert "The task uses multiple" in str(report.exc_info[1]) + + +@pytest.mark.end_to_end() +def test_deprecation_warning_for_strings_in_depends_on(runner, tmp_path): + source = """ + import pytask + from pathlib import Path + + @pytask.mark.depends_on("in.txt") + @pytask.mark.produces("out.txt") + def task_write_text(depends_on, produces): + produces.touch() + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("in.txt").touch() + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert "FutureWarning" in result.output + assert "Using strings to specify a dependency" in result.output + assert "Using strings to specify a product" in result.output diff --git a/tests/test_mark.py b/tests/test_mark.py index 32d468b1a..422b7bf09 100644 --- a/tests/test_mark.py +++ b/tests/test_mark.py @@ -1,5 +1,6 @@ from __future__ import annotations +import subprocess import sys import textwrap @@ -354,3 +355,23 @@ def task_example(): assert result.exit_code == ExitCode.OK assert "1 Succeeded" in result.output assert "Warnings" in result.output + + +@pytest.mark.end_to_end() +def test_deprecation_warnings_for_decorators(tmp_path): + source = """ + import pytask + + @pytask.mark.depends_on("in.txt") + @pytask.mark.produces("out.txt") + def task_write_text(depends_on, produces): + ... + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("in.txt").touch() + + result = subprocess.run( + ("pytest", tmp_path.joinpath("task_module.py").as_posix()), capture_output=True + ) + assert b"DeprecationWarning: '@pytask.mark.depends_on'" in result.stdout + assert b"DeprecationWarning: '@pytask.mark.produces'" in result.stdout