From 5fd0f29a9c34d5c0c302229a27f9a4a0e2266a53 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 13 Jul 2023 15:31:50 +0200 Subject: [PATCH 1/9] Add support for namedtuples. --- docs/source/changes.md | 2 ++ src/_pytask/collect_utils.py | 36 ++++++++++++---------- src/_pytask/task_utils.py | 29 ++++++++++++----- tests/test_task.py | 60 ++++++++++++++++++++++++++++++++++++ tests/test_task_utils.py | 32 +++++++++++++++++++ 5 files changed, 136 insertions(+), 23 deletions(-) diff --git a/docs/source/changes.md b/docs/source/changes.md index 7998f5748..52a4e87fc 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -17,6 +17,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`395` refactors all occurrences of pybaum to {mod}`_pytask.tree_util`. - {pull}`396` replaces pybaum with optree and adds paths to the name of {class}`pytask.PythonNode`'s allowing for better hashing. +- {class}`397` adds support for {class}`typing.NamedTuple` in + `@pytask.mark.task(kwargs=...)`. ## 0.3.2 - 2023-06-07 diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 696c4c055..37791ab85 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -321,13 +321,18 @@ def parse_products_from_task_function( has_annotation = False out = {} + # Parse products from decorators. if has_mark(obj, "produces"): has_produces_decorator = True nodes = parse_nodes(session, path, name, obj, produces) out = {"produces": nodes} task_kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {} - if "produces" in task_kwargs: + signature_defaults = parse_keyword_arguments_from_signature_defaults(obj) + kwargs = {**signature_defaults, **task_kwargs} + + # Parse products from task decorated with @task and that uses produces. + if "produces" in kwargs: collected_products = tree_map_with_path( lambda p, x: _collect_product( session, @@ -336,12 +341,13 @@ def parse_products_from_task_function( NodeInfo(arg_name="produces", path=p, value=x), is_string_allowed=True, ), - task_kwargs["produces"], + kwargs["produces"], ) out = {"produces": collected_products} parameters = inspect.signature(obj).parameters + # Parse products from default arguments if not has_mark(obj, "task") and "produces" in parameters: parameter = parameters["produces"] if parameter.default is not parameter.empty: @@ -363,20 +369,18 @@ def parse_products_from_task_function( if parameters_with_product_annot: has_annotation = True for parameter_name in parameters_with_product_annot: - parameter = parameters[parameter_name] - if parameter.default is not parameter.empty: - # Use _collect_new_node to not collect strings. - collected_products = tree_map_with_path( - lambda p, x: _collect_product( - session, - path, - name, - NodeInfo(parameter_name, p, x), # noqa: B023 - is_string_allowed=False, - ), - parameter.default, - ) - out = {parameter_name: collected_products} + # Use _collect_new_node to not collect strings. + collected_products = tree_map_with_path( + lambda p, x: _collect_product( + session, + path, + name, + NodeInfo(parameter_name, p, x), # noqa: B023 + is_string_allowed=False, + ), + kwargs[parameter_name], + ) + out = {parameter_name: collected_products} if ( sum( diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py index 750605a51..a9ccb014c 100644 --- a/src/_pytask/task_utils.py +++ b/src/_pytask/task_utils.py @@ -7,6 +7,7 @@ from typing import Any from typing import Callable +import attrs from _pytask.mark import Mark from _pytask.models import CollectionMetadata from _pytask.shared import find_duplicates @@ -143,24 +144,38 @@ def _parse_tasks_with_preliminary_names( def _parse_task(task: Callable[..., Any]) -> tuple[str, Callable[..., Any]]: """Parse a single task.""" - name = task.pytask_meta.name # type: ignore[attr-defined] - if name is None and task.__name__ == "_": + meta = task.pytask_meta # type: ignore[attr-defined] + + if meta.name is None and task.__name__ == "_": raise ValueError( "A task function either needs 'name' passed by the ``@pytask.mark.task`` " "decorator or the function name of the task function must not be '_'." ) - parsed_name = task.__name__ if name is None else name + parsed_name = task.__name__ if meta.name is None else meta.name + parsed_kwargs = _parse_task_kwargs(meta.kwargs) signature_kwargs = parse_keyword_arguments_from_signature_defaults(task) - task.pytask_meta.kwargs = { # type: ignore[attr-defined] - **task.pytask_meta.kwargs, # type: ignore[attr-defined] - **signature_kwargs, - } + meta.kwargs = {**signature_kwargs, **parsed_kwargs} return parsed_name, task +def _parse_task_kwargs(kwargs: Any) -> dict[str, Any]: + """Parse task kwargs.""" + if isinstance(kwargs, dict): + return kwargs + # Handle namedtuples. + if callable(getattr(kwargs, "_asdict", None)): + return kwargs._asdict() + if attrs.has(type(kwargs)): + return attrs.asdict(kwargs) + raise ValueError( + "'@pytask.mark.task(kwargs=...) needs to be a dictionary, namedtuple or an " + "instance of an attrs class." + ) + + def parse_keyword_arguments_from_signature_defaults( task: Callable[..., Any] ) -> dict[str, Any]: diff --git a/tests/test_task.py b/tests/test_task.py index ccbadfaef..de5ff8bd3 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -451,3 +451,63 @@ def task_example(): pass tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.FAILED + + +@pytest.mark.end_to_end() +def test_task_receives_namedtuple(runner, tmp_path): + source = """ + import pytask + from typing_extensions import NamedTuple, Annotated + from pathlib import Path + from pytask import Product, PythonNode + + class Args(NamedTuple): + path_in: Path + arg: str + path_out: Path + + + args = Args(Path("input.txt"), "world!", Path("output.txt")) + + @pytask.mark.task(kwargs=args) + def task_example( + path_in: Path, + arg: Annotated[str, PythonNode(hash=True)], + path_out: Annotated[Path, Product] + ) -> None: + path_out.write_text(path_in.read_text() + " " + arg) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("input.txt").write_text("hello") + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("output.txt").read_text() == "Hello world!" + + +@pytest.mark.end_to_end() +def test_task_kwargs_overwrite_default_arguments(runner, tmp_path): + source = """ + import pytask + from pytask import Product + from pathlib import Path + from typing_extensions import Annotated + + @pytask.mark.task(kwargs={ + "in_path": Path("in.txt"), "addition": "world!", "out_path": Path("out.txt") + }) + def task_example( + in_path: Path = Path("not_used_in.txt"), + addition: str = "planet!", + out_path: Annotated[Path, Product] = Path("not_used_out.txt"), + ) -> None: + out_path.write_text(in_path.read_text() + addition) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("in.txt").write_text("Hello ") + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("out.txt").read_text() == "Hello world!" + assert not tmp_path.joinpath("not_used_out.txt").exists() diff --git a/tests/test_task_utils.py b/tests/test_task_utils.py index 779cccd13..f3b51249d 100644 --- a/tests/test_task_utils.py +++ b/tests/test_task_utils.py @@ -1,7 +1,12 @@ from __future__ import annotations +from contextlib import ExitStack as does_not_raise # noqa: N813 +from typing import NamedTuple + import pytest from _pytask.task_utils import _arg_value_to_id_component +from _pytask.task_utils import _parse_task_kwargs +from attrs import define @pytest.mark.unit() @@ -24,3 +29,30 @@ def test_arg_value_to_id_component(arg_name, arg_value, i, id_func, expected): result = _arg_value_to_id_component(arg_name, arg_value, i, id_func) assert result == expected + + +class ExampleNT(NamedTuple): + a: int = 1 + + +@define +class ExampleAttrs: + b: str = "wonderful" + + +@pytest.mark.unit() +@pytest.mark.parametrize( + ("kwargs", "expectation", "expected"), + [ + ({"hello": 1}, does_not_raise(), {"hello": 1}), + (ExampleNT(), does_not_raise(), {"a": 1}), + (ExampleNT, pytest.raises(TypeError, match="ExampleNT._asdict()"), None), + (ExampleAttrs(), does_not_raise(), {"b": "wonderful"}), + (ExampleAttrs, pytest.raises(ValueError, match="@pytask.mark.task"), None), + (1, pytest.raises(ValueError, match="@pytask.mark.task"), None), + ], +) +def test_parse_task_kwargs(kwargs, expectation, expected): + with expectation: + result = _parse_task_kwargs(kwargs) + assert result == expected From 3e5de89c4de2065a4d18f6d467f2291a94a36daf Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 13 Jul 2023 15:33:06 +0200 Subject: [PATCH 2/9] Fix. --- docs/source/changes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/changes.md b/docs/source/changes.md index 52a4e87fc..18c6c6c6d 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -17,7 +17,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`395` refactors all occurrences of pybaum to {mod}`_pytask.tree_util`. - {pull}`396` replaces pybaum with optree and adds paths to the name of {class}`pytask.PythonNode`'s allowing for better hashing. -- {class}`397` adds support for {class}`typing.NamedTuple` in +- {class}`397` adds support for {class}`typing.NamedTuple` and attrs classes in `@pytask.mark.task(kwargs=...)`. ## 0.3.2 - 2023-06-07 From fd63a48b88351917f93983d014ade39df59e2680 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 14 Jul 2023 13:16:15 +0200 Subject: [PATCH 3/9] Fix errors and simplify collection. --- src/_pytask/collect_utils.py | 134 ++++++++++++++++++++--------------- src/_pytask/task_utils.py | 9 --- tests/test_collect.py | 12 ++-- tests/test_task.py | 2 +- 4 files changed, 84 insertions(+), 73 deletions(-) diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 37791ab85..80302d89c 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -1,7 +1,6 @@ """This module provides utility functions for :mod:`_pytask.collect`.""" from __future__ import annotations -import inspect import itertools import uuid from pathlib import Path @@ -79,9 +78,15 @@ def parse_nodes( session: Session, path: Path, name: str, obj: Any, parser: Callable[..., Any] ) -> Any: """Parse nodes from object.""" + arg_name = parser.__name__ objects = _extract_nodes_from_function_markers(obj, parser) - nodes = _convert_objects_to_node_dictionary(objects, parser.__name__) - nodes = tree_map(lambda x: _collect_old_dependencies(session, path, name, x), nodes) + nodes = _convert_objects_to_node_dictionary(objects, arg_name) + nodes = tree_map( + lambda x: _collect_decorator_nodes( + session, path, name, NodeInfo(arg_name, (), x) + ), + nodes, + ) return nodes @@ -211,10 +216,23 @@ def _merge_dictionaries(list_of_dicts: list[dict[Any, Any]]) -> dict[Any, Any]: return out -def parse_dependencies_from_task_function( +_ERROR_MULTIPLE_DEPENDENCY_DEFINITIONS = """"Dependencies are defined via \ +'@pytask.mark.depends_on' and as default arguments for the argument 'depends_on'. Use \ +only one way and not both. + +Hint: You do not need to use 'depends_on' since pytask v0.4. Every function argument \ +that is not a product is treated as a dependency. Read more about dependencies in the \ +documentation: https://tinyurl.com/yrezszr4. +""" + + +def parse_dependencies_from_task_function( # noqa: C901 session: Session, path: Path, name: str, obj: Any ) -> dict[str, Any]: """Parse dependencies from task function.""" + has_depends_on_decorator = False + has_depends_on_argument = False + if has_mark(obj, "depends_on"): nodes = parse_nodes(session, path, name, obj, depends_on) return {"depends_on": nodes} @@ -224,14 +242,30 @@ def parse_dependencies_from_task_function( kwargs = {**signature_defaults, **task_kwargs} kwargs.pop("produces", None) + dependencies = {} + # Parse products from task decorated with @task and that uses produces. + if "depends_on" in kwargs: + has_depends_on_argument = True + dependencies["depends_on"] = tree_map( + lambda x: _collect_decorator_nodes( + session, path, name, NodeInfo(arg_name="depends_on", path=(), value=x) + ), + kwargs["depends_on"], + ) + + if has_depends_on_decorator and has_depends_on_argument: + raise NodeNotCollectedError(_ERROR_MULTIPLE_PRODUCT_DEFINITIONS) + parameters_with_product_annot = _find_args_with_product_annotation(obj) parameters_with_node_annot = _find_args_with_node_annotation(obj) - dependencies = {} for parameter_name, value in kwargs.items(): if parameter_name in parameters_with_product_annot: continue + if parameter_name == "depends_on": + continue + if parameter_name in parameters_with_node_annot: def _evolve(x: Any) -> Any: @@ -316,8 +350,7 @@ def parse_products_from_task_function( """ has_produces_decorator = False - has_task_decorator = False - has_signature_default = False + has_produces_argument = False has_annotation = False out = {} @@ -333,6 +366,7 @@ def parse_products_from_task_function( # Parse products from task decorated with @task and that uses produces. if "produces" in kwargs: + has_produces_argument = True collected_products = tree_map_with_path( lambda p, x: _collect_product( session, @@ -345,52 +379,26 @@ def parse_products_from_task_function( ) out = {"produces": collected_products} - parameters = inspect.signature(obj).parameters - - # Parse products from default arguments - if not has_mark(obj, "task") and "produces" in parameters: - parameter = parameters["produces"] - if parameter.default is not parameter.empty: - has_signature_default = True - # Use _collect_new_node to not collect strings. - collected_products = tree_map_with_path( - lambda p, x: _collect_product( - session, - path, - name, - NodeInfo(arg_name="produces", path=p, value=x), - is_string_allowed=False, - ), - parameter.default, - ) - out = {"produces": collected_products} - parameters_with_product_annot = _find_args_with_product_annotation(obj) if parameters_with_product_annot: has_annotation = True for parameter_name in parameters_with_product_annot: - # Use _collect_new_node to not collect strings. - collected_products = tree_map_with_path( - lambda p, x: _collect_product( - session, - path, - name, - NodeInfo(parameter_name, p, x), # noqa: B023 - is_string_allowed=False, - ), - kwargs[parameter_name], - ) - out = {parameter_name: collected_products} + if parameter_name in kwargs: + # Use _collect_new_node to not collect strings. + collected_products = tree_map_with_path( + lambda p, x: _collect_product( + session, + path, + name, + NodeInfo(parameter_name, p, x), # noqa: B023 + is_string_allowed=False, + ), + kwargs[parameter_name], + ) + out = {parameter_name: collected_products} if ( - sum( - ( - has_produces_decorator, - has_task_decorator, - has_signature_default, - has_annotation, - ) - ) + sum((has_produces_decorator, has_produces_argument, has_annotation)) >= 2 # noqa: PLR2004 ): raise NodeNotCollectedError(_ERROR_MULTIPLE_PRODUCT_DEFINITIONS) @@ -416,8 +424,15 @@ def _find_args_with_product_annotation(func: Callable[..., Any]) -> list[str]: return args_with_product_annot -def _collect_old_dependencies( - session: Session, path: Path, name: str, node: str | Path +_ERROR_WRONG_TYPE_DECORATOR = """'@pytask.mark.depends_on', '@pytask.mark.produces', \ +and their function arguments can only accept values of type 'str' and 'pathlib.Path' \ +or the same values nested in tuples, lists, and dictionaries. Here, {node} has type \ +{node_type}. +""" + + +def _collect_decorator_nodes( + session: Session, path: Path, name: str, node_info: NodeInfo ) -> dict[str, MetaNode]: """Collect nodes for a task. @@ -427,22 +442,26 @@ def _collect_old_dependencies( If the node could not collected. """ + node = node_info.value + if not isinstance(node, (str, Path)): - raise ValueError( - "'@pytask.mark.depends_on' and '@pytask.mark.produces' can only accept " - "values of type 'str' and 'pathlib.Path' or the same values nested in " - f"tuples, lists, and dictionaries. Here, {node} has type {type(node)}." + raise NodeNotCollectedError( + _ERROR_WRONG_TYPE_DECORATOR.format(node=node, node_type=type(node)) ) if isinstance(node, str): node = Path(node) + node_info = node_info._replace(value=node) collected_node = session.hook.pytask_collect_node( - session=session, path=path, node_info=NodeInfo("produces", (), node) + session=session, path=path, node_info=node_info ) if collected_node is None: + kind = {"depends_on": "dependency", "produces": "product"}.get( + node_info.arg_name + ) raise NodeNotCollectedError( - f"{node!r} cannot be parsed as a dependency for task {name!r} in {path!r}." + f"{node!r} cannot be parsed as a {kind} for task {name!r} in {path!r}." ) return collected_node @@ -462,7 +481,7 @@ def _collect_dependencies( node = node_info.value collected_node = session.hook.pytask_collect_node( - session=session, path=path, node_info=node_info, node=node + session=session, path=path, node_info=node_info ) if collected_node is None: raise NodeNotCollectedError( @@ -509,9 +528,10 @@ def _collect_product( if isinstance(node, str): node = Path(node) + node_info = node_info._replace(value=node) collected_node = session.hook.pytask_collect_node( - session=session, path=path, node_info=node_info, node=node + session=session, path=path, node_info=node_info ) if collected_node is None: raise NodeNotCollectedError( diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py index a9ccb014c..cd5a1acef 100644 --- a/src/_pytask/task_utils.py +++ b/src/_pytask/task_utils.py @@ -114,15 +114,6 @@ def parse_collected_tasks_with_task_marker( else: collected_tasks[name] = [i[1] for i in parsed_tasks if i[0] == name][0] - # TODO: Remove when parsing dependencies and products from all arguments is - # implemented. - for task in collected_tasks.values(): - meta = task.pytask_meta # type: ignore[attr-defined] - for marker_name in ("depends_on", "produces"): - if marker_name in meta.kwargs: - value = meta.kwargs.pop(marker_name) - meta.markers.append(Mark(marker_name, (value,), {})) - return collected_tasks diff --git a/tests/test_collect.py b/tests/test_collect.py index 03bb85466..f0ba8fc5b 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -8,6 +8,7 @@ import pytest from _pytask.collect import _find_shortest_uniquely_identifiable_name_for_tasks from _pytask.collect import pytask_collect_node +from _pytask.exceptions import NodeNotCollectedError from _pytask.models import NodeInfo from pytask import cli from pytask import CollectionOutcome @@ -53,7 +54,7 @@ def task_with_non_path_dependency(): assert session.exit_code == ExitCode.COLLECTION_FAILED assert session.collection_reports[0].outcome == CollectionOutcome.FAIL exc_info = session.collection_reports[0].exc_info - assert isinstance(exc_info[1], ValueError) + assert isinstance(exc_info[1], NodeNotCollectedError) assert "'@pytask.mark.depends_on'" in str(exc_info[1]) @@ -74,7 +75,7 @@ def task_with_non_path_dependency(): assert session.exit_code == ExitCode.COLLECTION_FAILED assert session.collection_reports[0].outcome == CollectionOutcome.FAIL exc_info = session.collection_reports[0].exc_info - assert isinstance(exc_info[1], ValueError) + assert isinstance(exc_info[1], NodeNotCollectedError) assert "'@pytask.mark.depends_on'" in str(exc_info[1]) @@ -326,7 +327,7 @@ def task_write_text(produces="out.txt"): @pytest.mark.end_to_end() -def test_collect_string_product_as_function_default_fails(tmp_path): +def test_collect_string_product_as_function_default(tmp_path): source = """ import pytask @@ -335,9 +336,8 @@ def task_write_text(produces="out.txt"): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) session = main({"paths": tmp_path}) - report = session.collection_reports[0] - assert report.outcome == CollectionOutcome.FAIL - assert "If you use 'produces'" in str(report.exc_info[1]) + assert session.exit_code == ExitCode.OK + assert tmp_path.joinpath("out.txt").exists() @pytest.mark.end_to_end() diff --git a/tests/test_task.py b/tests/test_task.py index de5ff8bd3..b5d411fa8 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -478,7 +478,7 @@ def task_example( path_out.write_text(path_in.read_text() + " " + arg) """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - tmp_path.joinpath("input.txt").write_text("hello") + tmp_path.joinpath("input.txt").write_text("Hello") result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.OK From 9e4658e6602bd78569ffd9d177335e594f74f5a8 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 16 Jul 2023 20:55:14 +0200 Subject: [PATCH 4/9] Fix dependency collection and add test. --- src/_pytask/collect_utils.py | 17 +++++++++++------ tests/test_collect.py | 27 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 80302d89c..7004f5de9 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -216,9 +216,13 @@ def _merge_dictionaries(list_of_dicts: list[dict[Any, Any]]) -> dict[Any, Any]: return out -_ERROR_MULTIPLE_DEPENDENCY_DEFINITIONS = """"Dependencies are defined via \ -'@pytask.mark.depends_on' and as default arguments for the argument 'depends_on'. Use \ -only one way and not both. +_ERROR_MULTIPLE_DEPENDENCY_DEFINITIONS = """The task uses multiple ways to define \ +dependencies. Dependencies should be defined with either + +- '@pytask.mark.depends_on' +- as default arguments for the argument 'depends_on' + +Use only one of the two ways. Hint: You do not need to use 'depends_on' since pytask v0.4. Every function argument \ that is not a product is treated as a dependency. Read more about dependencies in the \ @@ -232,17 +236,18 @@ def parse_dependencies_from_task_function( # noqa: C901 """Parse dependencies from task function.""" has_depends_on_decorator = False has_depends_on_argument = False + dependencies = {} if has_mark(obj, "depends_on"): + has_depends_on_decorator = True nodes = parse_nodes(session, path, name, obj, depends_on) - return {"depends_on": nodes} + dependencies["depends_on"] = nodes task_kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {} signature_defaults = parse_keyword_arguments_from_signature_defaults(obj) kwargs = {**signature_defaults, **task_kwargs} kwargs.pop("produces", None) - dependencies = {} # Parse products from task decorated with @task and that uses produces. if "depends_on" in kwargs: has_depends_on_argument = True @@ -254,7 +259,7 @@ def parse_dependencies_from_task_function( # noqa: C901 ) if has_depends_on_decorator and has_depends_on_argument: - raise NodeNotCollectedError(_ERROR_MULTIPLE_PRODUCT_DEFINITIONS) + raise NodeNotCollectedError(_ERROR_MULTIPLE_DEPENDENCY_DEFINITIONS) parameters_with_product_annot = _find_args_with_product_annotation(obj) parameters_with_node_annot = _find_args_with_node_annotation(obj) diff --git a/tests/test_collect.py b/tests/test_collect.py index f0ba8fc5b..bb90f5d2b 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -362,3 +362,30 @@ def task_example( report = session.collection_reports[0] assert report.outcome == CollectionOutcome.FAIL assert "The task uses multiple ways" in str(report.exc_info[1]) + + +@pytest.mark.end_to_end() +def test_depends_on_cannot_mix_different_definitions(tmp_path): + source = """ + import pytask + from typing_extensions import Annotated + from pytask import Product + from pathlib import Path + + @pytask.mark.depends_on("input_1.txt") + def task_example( + depends_on: Path = "input_2.txt", + path: Annotated[Path, Product] = Path("out.txt") + ): + ... + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("input_1.txt").touch() + tmp_path.joinpath("input_2.txt").touch() + session = main({"paths": tmp_path}) + + assert session.exit_code == ExitCode.COLLECTION_FAILED + assert len(session.tasks) == 0 + report = session.collection_reports[0] + assert report.outcome == CollectionOutcome.FAIL + assert "The task uses multiple" in str(report.exc_info[1]) From 766bb308b7eb9aa0cd663ba5293dfcfec77ed9cd Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 16 Jul 2023 20:58:36 +0200 Subject: [PATCH 5/9] Fix remote test. --- tests/test_task_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_task_utils.py b/tests/test_task_utils.py index f3b51249d..2debabaeb 100644 --- a/tests/test_task_utils.py +++ b/tests/test_task_utils.py @@ -46,7 +46,7 @@ class ExampleAttrs: [ ({"hello": 1}, does_not_raise(), {"hello": 1}), (ExampleNT(), does_not_raise(), {"a": 1}), - (ExampleNT, pytest.raises(TypeError, match="ExampleNT._asdict()"), None), + (ExampleNT, pytest.raises(TypeError, match="_asdict() missing 1"), None), (ExampleAttrs(), does_not_raise(), {"b": "wonderful"}), (ExampleAttrs, pytest.raises(ValueError, match="@pytask.mark.task"), None), (1, pytest.raises(ValueError, match="@pytask.mark.task"), None), From 832ff3099f5c45de30e665dd740084c9a6c087cb Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 16 Jul 2023 21:04:39 +0200 Subject: [PATCH 6/9] Fix test. --- tests/test_task_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_task_utils.py b/tests/test_task_utils.py index 2debabaeb..36b2b2086 100644 --- a/tests/test_task_utils.py +++ b/tests/test_task_utils.py @@ -46,7 +46,7 @@ class ExampleAttrs: [ ({"hello": 1}, does_not_raise(), {"hello": 1}), (ExampleNT(), does_not_raise(), {"a": 1}), - (ExampleNT, pytest.raises(TypeError, match="_asdict() missing 1"), None), + (ExampleNT, pytest.raises(TypeError, match=r"(_asdict\(\) missing 1)"), None), (ExampleAttrs(), does_not_raise(), {"b": "wonderful"}), (ExampleAttrs, pytest.raises(ValueError, match="@pytask.mark.task"), None), (1, pytest.raises(ValueError, match="@pytask.mark.task"), None), From f00899f4e6ee65af04d0c186900491ad5039180f Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 16 Jul 2023 21:38:33 +0200 Subject: [PATCH 7/9] Fix for sqlalchemy >=v2.019. --- src/_pytask/parameters.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index d3fd83599..2ec31f53b 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -74,6 +74,11 @@ def _database_url_callback( ctx: Context, name: str, value: str | None # noqa: ARG001 ) -> URL: + """Check the url for the database.""" + # Since sqlalchemy v2.0.19, we need to shortcircuit here. + if value is None: + return None + try: return make_url(value) except ArgumentError: From d080e6c40cac41ddf175365fb16899d2ae080ff6 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 16 Jul 2023 21:52:05 +0200 Subject: [PATCH 8/9] xfail test on all OS. See #377. --- tests/test_live.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_live.py b/tests/test_live.py index abcfbacf7..4fe76273c 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -1,7 +1,6 @@ from __future__ import annotations import re -import sys import textwrap import pytest @@ -188,7 +187,7 @@ def test_live_execution_displays_subset_of_table(capsys, tmp_path, n_entries_in_ @pytest.mark.unit() -@pytest.mark.xfail(sys.platform == "darwin", reason="See #377.") +@pytest.mark.xfail(reason="See #377.") def test_live_execution_skips_do_not_crowd_out_displayed_tasks(capsys, tmp_path): path = tmp_path.joinpath("task_module.py") task = Task(base_name="task_example", path=path, function=lambda x: x) From 2490270e1682dbf79819209c8e180c335997d926 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 16 Jul 2023 22:13:06 +0200 Subject: [PATCH 9/9] make error messages more pretty. --- src/_pytask/collect_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 7004f5de9..19c4d924e 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -219,10 +219,10 @@ def _merge_dictionaries(list_of_dicts: list[dict[Any, Any]]) -> dict[Any, Any]: _ERROR_MULTIPLE_DEPENDENCY_DEFINITIONS = """The task uses multiple ways to define \ dependencies. Dependencies should be defined with either -- '@pytask.mark.depends_on' -- as default arguments for the argument 'depends_on' +- '@pytask.mark.depends_on' and a 'depends_on' function argument. +- as default value for the function argument 'depends_on'. -Use only one of the two ways. +Use only one of the two ways! Hint: You do not need to use 'depends_on' since pytask v0.4. Every function argument \ that is not a product is treated as a dependency. Read more about dependencies in the \