From 82366dbcc2936c2afd90be63771b1abe9279a62f Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 28 May 2023 10:50:46 +0200 Subject: [PATCH 1/7] Started PR. --- docs/source/changes.md | 5 +++ tests/test_collect.py | 99 +++++++++++++++++++++++++----------------- 2 files changed, 63 insertions(+), 41 deletions(-) diff --git a/docs/source/changes.md b/docs/source/changes.md index f0b23b6e3..85542ec84 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -5,6 +5,11 @@ chronological order. Releases follow [semantic versioning](https://semver.org/) releases are available on [PyPI](https://pypi.org/project/pytask) and [Anaconda.org](https://anaconda.org/conda-forge/pytask). +## 0.4.0 - 2023-xx-xx + +- {pull}`384` allows to parse dependencies from every function argument if `depends_on` + is not present. + ## 0.3.2 - 2023-xx-xx - {pull}`345` updates the version numbers in animations. diff --git a/tests/test_collect.py b/tests/test_collect.py index 9a10831a5..d5f7209b4 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -37,46 +37,6 @@ def task_write_text(depends_on, produces): assert tmp_path.joinpath("out.txt").read_text() == "Relative paths work." -@pytest.mark.end_to_end() -def test_collect_tasks_from_modules_with_the_same_name(tmp_path): - """We need to check that task modules can have the same name. See #373 and #374.""" - tmp_path.joinpath("a").mkdir() - tmp_path.joinpath("b").mkdir() - tmp_path.joinpath("a", "task_module.py").write_text("def task_a(): pass") - tmp_path.joinpath("b", "task_module.py").write_text("def task_a(): pass") - session = main({"paths": tmp_path}) - assert len(session.collection_reports) == 2 - assert all( - report.outcome == CollectionOutcome.SUCCESS - for report in session.collection_reports - ) - assert { - report.node.function.__module__ for report in session.collection_reports - } == {"a.task_module", "b.task_module"} - - -@pytest.mark.end_to_end() -def test_collect_module_name(tmp_path): - """We need to add a task module to the sys.modules. See #373 and #374.""" - source = """ - # without this import, everything works fine - from __future__ import annotations - - import dataclasses - - @dataclasses.dataclass - class Data: - x: int - - def task_my_task(): - pass - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) - outcome = session.collection_reports[0].outcome - assert outcome == CollectionOutcome.SUCCESS - - @pytest.mark.end_to_end() def test_collect_filepathnode_with_unknown_type(tmp_path): """If a node cannot be parsed because unknown type, raise an error.""" @@ -129,7 +89,7 @@ def task_1(depends_on, produces): @pytest.mark.end_to_end() @pytest.mark.parametrize("path_extension", ["", "task_module.py"]) -def test_collect_same_test_different_ways(tmp_path, path_extension): +def test_collect_same_task_different_ways(tmp_path, path_extension): tmp_path.joinpath("task_module.py").write_text("def task_passes(): pass") session = main({"paths": tmp_path.joinpath(path_extension)}) @@ -270,3 +230,60 @@ def test_find_shortest_uniquely_identifiable_names_for_tasks(tmp_path): result = _find_shortest_uniquely_identifiable_name_for_tasks(tasks) assert result == expected + + +def test_collect_dependencies_from_args_if_depends_on_is_missing(tmp_path): + source = """ + from pathlib import Path + + def task_example(path_in = Path("in.txt"), produces = Path("out.txt")): + produces.write_text(depends_on.read_text()) + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("in.txt").write_text("hello") + + session = main({"paths": tmp_path}) + + assert session.exit_code == ExitCode.OK + assert len(session.tasks) == 1 + assert session.tasks[0].depends_on[0].path == tmp_path.joinpath("in.txt") + + +@pytest.mark.end_to_end() +def test_collect_tasks_from_modules_with_the_same_name(tmp_path): + """We need to check that task modules can have the same name. See #373 and #374.""" + tmp_path.joinpath("a").mkdir() + tmp_path.joinpath("b").mkdir() + tmp_path.joinpath("a", "task_module.py").write_text("def task_a(): pass") + tmp_path.joinpath("b", "task_module.py").write_text("def task_a(): pass") + session = main({"paths": tmp_path}) + assert len(session.collection_reports) == 2 + assert all( + report.outcome == CollectionOutcome.SUCCESS + for report in session.collection_reports + ) + assert { + report.node.function.__module__ for report in session.collection_reports + } == {"a.task_module", "b.task_module"} + + +@pytest.mark.end_to_end() +def test_collect_module_name(tmp_path): + """We need to add a task module to the sys.modules. See #373 and #374.""" + source = """ + # without this import, everything works fine + from __future__ import annotations + + import dataclasses + + @dataclasses.dataclass + class Data: + x: int + + def task_my_task(): + pass + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + session = main({"paths": tmp_path}) + outcome = session.collection_reports[0].outcome + assert outcome == CollectionOutcome.SUCCESS From 39fadd6814cf793dc13d5875b4f0f2298d3b9269 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Tue, 27 Jun 2023 20:58:56 +0200 Subject: [PATCH 2/7] Add PythonNode, remove task.kwargs, make depends_on and produces mandatory as func args when decorator is used. --- .../defining_dependencies_products.md | 26 ++++----- src/_pytask/collect.py | 19 ++++--- src/_pytask/collect_utils.py | 53 ++++++++++++++++++- src/_pytask/dag.py | 5 +- src/_pytask/execute.py | 13 ++--- src/_pytask/nodes.py | 19 ++++++- src/_pytask/report.py | 1 - src/_pytask/task_utils.py | 7 ++- tests/test_build.py | 1 + tests/test_clean.py | 4 +- tests/test_collect.py | 31 +++++------ tests/test_database.py | 2 +- tests/test_dry_run.py | 4 +- tests/test_graph.py | 2 +- tests/test_pybaum.py | 2 + tests/test_skipping.py | 5 +- tests/test_task.py | 12 +++++ 17 files changed, 146 insertions(+), 60 deletions(-) diff --git a/docs/source/tutorials/defining_dependencies_products.md b/docs/source/tutorials/defining_dependencies_products.md index 98086b5ce..d245b2615 100644 --- a/docs/source/tutorials/defining_dependencies_products.md +++ b/docs/source/tutorials/defining_dependencies_products.md @@ -4,8 +4,8 @@ To ensure pytask executes all tasks in the correct order, define which dependenc required and which products are produced by a task. :::{important} -If you do not specify dependencies and products as explained below, pytask will not be able -to build a graph, a {term}`DAG`, and will not be able to execute all tasks in the +If you do not specify dependencies and products as explained below, pytask will not be +able to build a graph, a {term}`DAG`, and will not be able to execute all tasks in the project correctly! ::: @@ -19,15 +19,16 @@ def task_create_random_data(produces): ... ``` -The {func}`@pytask.mark.produces ` marker attaches a -product to a task which is a {class}`pathlib.Path` to file. After the task has finished, -pytask will check whether the file exists. +The {func}`@pytask.mark.produces ` marker attaches a product to a +task which is a {class}`pathlib.Path` to file. After the task has finished, pytask will +check whether the file exists. -Optionally, you can use `produces` as an argument of the task function and get access to -the same path inside the task function. +Add `produces` as an argument of the task function to get access to the same path inside +the task function. :::{tip} -If you do not know about {mod}`pathlib` check out [^id3] and [^id4]. The module is beneficial for handling paths conveniently and across platforms. +If you do not know about {mod}`pathlib` check out [^id3] and [^id4]. The module is +beneficial for handling paths conveniently and across platforms. ::: ## Dependencies @@ -44,7 +45,7 @@ def task_plot_data(depends_on, produces): ... ``` -Use `depends_on` as a function argument to work with the dependency path and, for +Add `depends_on` as a function argument to work with the path of the dependency and, for example, load the data. ## Conversion @@ -61,9 +62,6 @@ def task_create_random_data(produces): ... ``` -If you use `depends_on` or `produces` as arguments for the task function, you will have -access to the paths of the targets as {class}`pathlib.Path`. - ## Multiple dependencies and products The easiest way to attach multiple dependencies or products to a task is to pass a @@ -108,7 +106,9 @@ Why does pytask recommend dictionaries and convert lists, tuples, or other iterators to dictionaries? First, dictionaries with positions as keys behave very similarly to lists. -Secondly, dictionaries use keys instead of positions that are more verbose and descriptive and do not assume a fixed ordering. Both attributes are especially desirable in complex projects. +Secondly, dictionaries use keys instead of positions that are more verbose and +descriptive and do not assume a fixed ordering. Both attributes are especially desirable +in complex projects. ## Multiple decorators diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index dcb22df3e..08d24c493 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -12,6 +12,7 @@ from typing import Iterable from _pytask.collect_utils import depends_on +from _pytask.collect_utils import parse_dependencies_from_task_function from _pytask.collect_utils import parse_nodes from _pytask.collect_utils import produces from _pytask.config import hookimpl @@ -22,6 +23,7 @@ from _pytask.exceptions import CollectionError from _pytask.mark_utils import has_mark from _pytask.nodes import FilePathNode +from _pytask.nodes import PythonNode from _pytask.nodes import Task from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import count_outcomes @@ -167,11 +169,17 @@ def pytask_collect_task( """ if (name.startswith("task_") or has_mark(obj, "task")) and callable(obj): - dependencies = parse_nodes(session, path, name, obj, depends_on) + if has_mark(obj, "depends_on"): + nodes = parse_nodes(session, path, name, obj, depends_on) + dependencies = {"depends_on": nodes} + else: + dependencies = parse_dependencies_from_task_function( + session, path, name, obj + ) + products = parse_nodes(session, path, name, obj, produces) markers = obj.pytask_meta.markers if hasattr(obj, "pytask_meta") else [] - kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {} # Get the underlying function to avoid having different states of the function, # e.g. due to pytask_meta, in different layers of the wrapping. @@ -184,7 +192,6 @@ def pytask_collect_task( depends_on=dependencies, produces=products, markers=markers, - kwargs=kwargs, ) return None @@ -205,7 +212,7 @@ def pytask_collect_task( @hookimpl(trylast=True) def pytask_collect_node( session: Session, path: Path, node: str | Path -) -> FilePathNode | None: +) -> FilePathNode | PythonNode: """Collect a node of a task as a :class:`pytask.nodes.FilePathNode`. Strings are assumed to be paths. This might be a strict assumption, but since this @@ -226,8 +233,6 @@ def pytask_collect_node( handled by this function. """ - if isinstance(node, str): - node = Path(node) if isinstance(node, Path): if not node.is_absolute(): node = path.parent.joinpath(node) @@ -246,7 +251,7 @@ def pytask_collect_node( raise ValueError(_TEMPLATE_ERROR.format(node, case_sensitive_path)) return FilePathNode.from_path(node) - return None + return PythonNode(value=node) def _not_ignored_paths( diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index ca15e624a..780bb8af8 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -12,7 +12,9 @@ from _pytask.exceptions import NodeNotCollectedError from _pytask.mark_utils import remove_marks +from _pytask.nodes import PythonNode from _pytask.shared import find_duplicates +from _pytask.task_utils import parse_keyword_arguments_from_signature_defaults from attrs import define from attrs import field from pybaum.tree_util import tree_map @@ -23,7 +25,12 @@ from _pytask.nodes import MetaNode -__all__ = ["depends_on", "parse_nodes", "produces"] +__all__ = [ + "depends_on", + "parse_dependencies_from_task_function", + "parse_nodes", + "produces", +] def depends_on( @@ -220,6 +227,16 @@ def _collect_node( If the node could not collected. """ + if not isinstance(node, (str, Path)): + raise ValueError( + "'@pytask.mark.depends_on' and '@pytask.mark.produces' can only accept" + "values of type 'str' and 'pathlib.Path' or the same values nested in " + f"tuples, lists, and dictionaries. Here, {node} has type {type(node)}." + ) + + if isinstance(node, str): + node = Path(node) + collected_node = session.hook.pytask_collect_node( session=session, path=path, node=node ) @@ -230,3 +247,37 @@ def _collect_node( ) return collected_node + + +def parse_dependencies_from_task_function( + session: Session, path: Path, name: str, obj: Any +) -> dict[str, Any]: + """Parse dependencies from task function.""" + task_kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {} + signature_defaults = parse_keyword_arguments_from_signature_defaults(obj) + kwargs = {**signature_defaults, **task_kwargs} + kwargs.pop("produces", None) + + def _collect_node( + session: Session, path: Path, name: str, node: Any + ) -> dict[str, MetaNode]: + collected_node = session.hook.pytask_collect_node( + session=session, path=path, node=node + ) + if collected_node is None: + raise NodeNotCollectedError( + f"{node!r} cannot be parsed as a dependency for task " + f"{name!r} in {path!r}." + ) + return collected_node + + dependencies = {} + for name, value in kwargs.items(): + parsed_value = tree_map( + lambda x: _collect_node(session, path, name, x), value # noqa: B023 + ) + dependencies[name] = ( + PythonNode(value=None) if parsed_value is None else parsed_value + ) + + return dependencies diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py index c6acf0346..bbf199a5d 100644 --- a/src/_pytask/dag.py +++ b/src/_pytask/dag.py @@ -190,7 +190,10 @@ def _format_cycles(cycles: list[tuple[str, ...]]) -> str: "tree which shows which dependencies are missing for which tasks.\n\n{}" ) if IS_FILE_SYSTEM_CASE_SENSITIVE: - _TEMPLATE_ERROR += "\n\n(Hint: Sometimes case sensitivity is at fault.)" + _TEMPLATE_ERROR += ( + "\n\n(Hint: Your file-system is case-sensitive. Check the paths' " + "capitalization carefully.)" + ) def _check_if_root_nodes_are_available(dag: nx.DiGraph) -> None: diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index 1922e27aa..40e8e4b42 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -143,13 +143,14 @@ def pytask_execute_task(session: Session, task: Task) -> bool: if session.config["dry_run"]: raise WouldBeExecuted - kwargs = {**task.kwargs} + parameters = inspect.signature(task.function).parameters - func_arg_names = set(inspect.signature(task.function).parameters) - for arg_name in ("depends_on", "produces"): - if arg_name in func_arg_names: - attribute = getattr(task, arg_name) - kwargs[arg_name] = tree_map(lambda x: x.value, attribute) + kwargs = {} + for name, value in task.depends_on.items(): + kwargs[name] = tree_map(lambda x: x.value, value) + + if task.produces and "produces" in parameters: + kwargs["produces"] = tree_map(lambda x: x.value, task.produces) task.execute(**kwargs) return True diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index 2d330f02d..64db82849 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -51,8 +51,6 @@ class Task(MetaNode): """A list of products of task.""" markers: list[Mark] = field(factory=list) """A list of markers attached to the task function.""" - kwargs: dict[str, Any] = field(factory=dict) - """A dictionary with keyword arguments supplied to the task.""" _report_sections: list[tuple[str, str, str]] = field(factory=list) """Reports with entries for when, what, and content.""" attributes: dict[Any, Any] = field(factory=dict) @@ -108,3 +106,20 @@ def from_path(cls, path: Path) -> FilePathNode: if not path.is_absolute(): raise ValueError("FilePathNode must be instantiated from absolute path.") return cls(name=path.as_posix(), value=path, path=path) + + +@define(kw_only=True) +class PythonNode(MetaNode): + """The class for a node which is a Python object.""" + + value: Any + hash: bool = False # noqa: A003 + name: str = "" + + def __attrs_post_init__(self) -> None: + self.name = str(self.value) + + def state(self) -> str | None: + if self.hash: + return str(hash(self.value)) + return str(0) diff --git a/src/_pytask/report.py b/src/_pytask/report.py index 0bc0cf03d..ad9abe30d 100644 --- a/src/_pytask/report.py +++ b/src/_pytask/report.py @@ -37,7 +37,6 @@ def from_exception( exc_info: ExceptionInfo, node: MetaNode | None = None, ) -> CollectionReport: - exc_info = remove_internal_traceback_frames_from_exc_info(exc_info) return cls(outcome=outcome, node=node, exc_info=exc_info) diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py index ad541c9d1..750605a51 100644 --- a/src/_pytask/task_utils.py +++ b/src/_pytask/task_utils.py @@ -12,6 +12,9 @@ from _pytask.shared import find_duplicates +__all__ = ["parse_keyword_arguments_from_signature_defaults"] + + COLLECTED_TASKS: dict[Path, list[Callable[..., Any]]] = defaultdict(list) """A container for collecting tasks. @@ -149,7 +152,7 @@ def _parse_task(task: Callable[..., Any]) -> tuple[str, Callable[..., Any]]: parsed_name = task.__name__ if name is None else name - signature_kwargs = _parse_keyword_arguments_from_signature_defaults(task) + signature_kwargs = parse_keyword_arguments_from_signature_defaults(task) task.pytask_meta.kwargs = { # type: ignore[attr-defined] **task.pytask_meta.kwargs, # type: ignore[attr-defined] **signature_kwargs, @@ -158,7 +161,7 @@ def _parse_task(task: Callable[..., Any]) -> tuple[str, Callable[..., Any]]: return parsed_name, task -def _parse_keyword_arguments_from_signature_defaults( +def parse_keyword_arguments_from_signature_defaults( task: Callable[..., Any] ) -> dict[str, Any]: """Parse keyword arguments from signature defaults.""" diff --git a/tests/test_build.py b/tests/test_build.py index 96f158e6f..75d1d4e8d 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -38,6 +38,7 @@ def test_collection_failed(runner, tmp_path): @pytest.mark.end_to_end() def test_building_dag_failed(runner, tmp_path): + # TODO: Should maybe fail because depends_on and produces are not used. source = """ import pytask diff --git a/tests/test_clean.py b/tests/test_clean.py index a74752740..4c4c0bdd6 100644 --- a/tests/test_clean.py +++ b/tests/test_clean.py @@ -19,7 +19,7 @@ def project(tmp_path): @pytask.mark.depends_on("in.txt") @pytask.mark.produces("out.txt") - def task_write_text(produces): + def task_write_text(depends_on, produces): produces.write_text("a") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) @@ -40,7 +40,7 @@ def git_project(tmp_path): @pytask.mark.depends_on("in_tracked.txt") @pytask.mark.produces("out.txt") - def task_write_text(produces): + def task_write_text(depends_on, produces): produces.write_text("a") """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_collect.py b/tests/test_collect.py index d5f7209b4..1766765e1 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -3,13 +3,11 @@ import sys import textwrap import warnings -from contextlib import ExitStack as does_not_raise # noqa: N813 from pathlib import Path import pytest from _pytask.collect import _find_shortest_uniquely_identifiable_name_for_tasks from _pytask.collect import pytask_collect_node -from _pytask.exceptions import NodeNotCollectedError from pytask import cli from pytask import CollectionOutcome from pytask import ExitCode @@ -54,7 +52,7 @@ def task_with_non_path_dependency(): assert session.exit_code == ExitCode.COLLECTION_FAILED assert session.collection_reports[0].outcome == CollectionOutcome.FAIL exc_info = session.collection_reports[0].exc_info - assert isinstance(exc_info[1], NodeNotCollectedError) + assert isinstance(exc_info[1], ValueError) @pytest.mark.end_to_end() @@ -127,13 +125,12 @@ def test_collect_files_w_custom_file_name_pattern( @pytest.mark.unit() @pytest.mark.parametrize( - ("session", "path", "node", "expectation", "expected"), + ("session", "path", "node", "expected"), [ pytest.param( Session({"check_casing_of_paths": False}, None), Path(), Path.cwd() / "text.txt", - does_not_raise(), Path.cwd() / "text.txt", id="test with absolute string path", ), @@ -141,19 +138,17 @@ def test_collect_files_w_custom_file_name_pattern( Session({"check_casing_of_paths": False}, None), Path(), 1, - does_not_raise(), - None, - id="test cannot collect node", + "1", + id="test with python node", ), ], ) -def test_pytask_collect_node(session, path, node, expectation, expected): - with expectation: - result = pytask_collect_node(session, path, node) - if result is None: - assert result is expected - else: - assert str(result.path) == str(expected) +def test_pytask_collect_node(session, path, node, expected): + result = pytask_collect_node(session, path, node) + if result is None: + assert result is expected + else: + assert str(result.value) == str(expected) @pytest.mark.unit() @@ -180,7 +175,7 @@ def test_pytask_collect_node_does_not_raise_error_if_path_is_not_normalized( task_path = tmp_path / "task_example.py" real_node = tmp_path / "text.txt" - collected_node = f"../{tmp_path.name}/text.txt" + collected_node = Path("..", tmp_path.name, "text.txt") if is_absolute: collected_node = tmp_path / collected_node @@ -237,7 +232,7 @@ def test_collect_dependencies_from_args_if_depends_on_is_missing(tmp_path): from pathlib import Path def task_example(path_in = Path("in.txt"), produces = Path("out.txt")): - produces.write_text(depends_on.read_text()) + produces.write_text(path_in.read_text()) """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("hello") @@ -246,7 +241,7 @@ def task_example(path_in = Path("in.txt"), produces = Path("out.txt")): assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 - assert session.tasks[0].depends_on[0].path == tmp_path.joinpath("in.txt") + assert session.tasks[0].depends_on["path_in"].path == tmp_path.joinpath("in.txt") @pytest.mark.end_to_end() diff --git a/tests/test_database.py b/tests/test_database.py index 36d831d47..db46e0e4a 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -19,7 +19,7 @@ def test_existence_of_hashes_in_db(tmp_path, runner): @pytask.mark.depends_on("in.txt") @pytask.mark.produces("out.txt") - def task_write(produces): + def task_write(depends_on, produces): produces.touch() """ task_path = tmp_path.joinpath("task_module.py") diff --git a/tests/test_dry_run.py b/tests/test_dry_run.py index d73a5ab69..13114c05a 100644 --- a/tests/test_dry_run.py +++ b/tests/test_dry_run.py @@ -33,7 +33,7 @@ def test_dry_run_w_subsequent_task(runner, tmp_path): @pytask.mark.depends_on("out.txt") @pytask.mark.produces("out_2.txt") - def task_example(produces): + def task_example(depends_on, produces): produces.touch() """ tmp_path.joinpath("task_example_second.py").write_text(textwrap.dedent(source)) @@ -80,7 +80,7 @@ def task_example(produces): @pytask.mark.depends_on("out.txt") @pytask.mark.produces("out_2.txt") - def task_example(produces): + def task_example(depends_on, produces): produces.touch() """ tmp_path.joinpath("task_example_second.py").write_text(textwrap.dedent(source_2)) diff --git a/tests/test_graph.py b/tests/test_graph.py index 38f6158ca..c7aecc590 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -86,7 +86,7 @@ def test_create_graph_via_task(tmp_path, runner, format_, layout, rankdir): import networkx as nx @pytask.mark.depends_on("input.txt") - def task_example(): pass + def task_example(depends_on): pass def task_create_graph(): dag = pytask.build_dag({{"paths": Path(__file__).parent}}) diff --git a/tests/test_pybaum.py b/tests/test_pybaum.py index 69ceee349..66ea24e60 100644 --- a/tests/test_pybaum.py +++ b/tests/test_pybaum.py @@ -50,6 +50,8 @@ def task_example(): 2: {0: tmp_path / "list_out.txt"}, 3: {"a": tmp_path / "dict_out.txt", "b": {"c": tmp_path / "dict_out_2.txt"}}, } + if decorator_name == "depends_on": + expected = {"depends_on": expected} assert products == expected diff --git a/tests/test_skipping.py b/tests/test_skipping.py index f7c1ad7c2..bb18d0fae 100644 --- a/tests/test_skipping.py +++ b/tests/test_skipping.py @@ -195,11 +195,10 @@ def test_if_skipif_decorator_is_applied_execute(tmp_path): @pytask.mark.skipif(False, reason="bla") @pytask.mark.produces("out.txt") def task_first(produces): - with open(produces, "w") as f: - f.write("hello world.") + produces.touch() @pytask.mark.depends_on("out.txt") - def task_second(): + def task_second(depends_on): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_task.py b/tests/test_task.py index 3313fdb7b..bf4c16a37 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -438,3 +438,15 @@ def task_func(i=i): assert session.exit_code == ExitCode.COLLECTION_FAILED assert isinstance(session.collection_reports[0].exc_info[1], ValueError) + + +def test_task_receives_unknown_kwarg(runner, tmp_path): + source = """ + import pytask + + @pytask.mark.task(kwargs={"i": 1}) + def task_example(): pass + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.FAILED From d705c4f49383f1e7112a93e73f0c74f61da2b9bc Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Tue, 27 Jun 2023 23:02:03 +0200 Subject: [PATCH 3/7] Fix pytask collect by skipping all non filenpath nodes. --- src/_pytask/collect_command.py | 31 ++++++++++++------ tests/test_collect_command.py | 59 +++++++++++++++++++++++++++++++--- 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 9bad3d365..977def55d 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -20,6 +20,7 @@ from _pytask.exceptions import ResolvingDependenciesError from _pytask.mark import select_by_keyword from _pytask.mark import select_by_mark +from _pytask.nodes import FilePathNode from _pytask.outcomes import ExitCode from _pytask.path import find_common_ancestor from _pytask.path import relative_to @@ -123,8 +124,16 @@ def _find_common_ancestor_of_all_nodes( for task in tasks: all_paths.append(task.path) if show_nodes: - all_paths.extend(x.path for x in tree_just_flatten(task.depends_on)) - all_paths.extend(x.path for x in tree_just_flatten(task.produces)) + all_paths.extend( + x.path + for x in tree_just_flatten(task.depends_on) + if isinstance(x, FilePathNode) + ) + all_paths.extend( + x.path + for x in tree_just_flatten(task.produces) + if isinstance(x, FilePathNode) + ) common_ancestor = find_common_ancestor(*all_paths, *paths) @@ -160,14 +169,14 @@ def _print_collected_tasks( Parameters ---------- - dictionary : Dict[Path, List["Task"]] + dictionary A dictionary with path on the first level, tasks on the second, dependencies and products on the third. - show_nodes : bool + show_nodes Indicator for whether dependencies and products should be displayed. - editor_url_scheme : str + editor_url_scheme The scheme to create an url. - common_ancestor : Path + common_ancestor The path common to all tasks and nodes. """ @@ -197,9 +206,13 @@ def _print_collected_tasks( ) if show_nodes: - for node in sorted( - tree_just_flatten(task.depends_on), key=lambda x: x.path - ): + file_path_nodes = [ + i + for i in tree_just_flatten(task.depends_on) + if isinstance(i, FilePathNode) + ] + sorted_nodes = sorted(file_path_nodes, key=lambda x: x.path) + for node in sorted_nodes: reduced_node_name = relative_to(node.path, common_ancestor) url_style = create_url_style_for_path(node.path, editor_url_scheme) task_branch.add( diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py index eba87f4cc..bfc23d927 100644 --- a/tests/test_collect_command.py +++ b/tests/test_collect_command.py @@ -7,6 +7,7 @@ import pytest from _pytask.collect_command import _find_common_ancestor_of_all_nodes from _pytask.collect_command import _print_collected_tasks +from _pytask.nodes import FilePathNode from attrs import define from pytask import cli from pytask import ExitCode @@ -50,6 +51,42 @@ def task_example(): assert "out.txt>" in captured +@pytest.mark.end_to_end() +def test_collect_task_new_interface(runner, tmp_path): + source = """ + import pytask + + @pytask.mark.depends_on("in.txt") + @pytask.mark.produces("out.txt") + def task_example(depends_on="in.txt", arg=1, produces="out.txt"): + pass + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("in.txt").touch() + + result = runner.invoke(cli, ["collect", tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + captured = result.output.replace("\n", "").replace(" ", "") + assert "" in captured + assert "" in captured + + result = runner.invoke(cli, ["collect", tmp_path.as_posix(), "--nodes"]) + + assert result.exit_code == ExitCode.OK + captured = result.output.replace("\n", "").replace(" ", "") + assert "" in captured + assert "" in captured + assert "" in captured + assert "" in captured + + @pytest.mark.end_to_end() def test_collect_task_in_root_dir(runner, tmp_path): source = """ @@ -313,7 +350,7 @@ def state(self): ... -def function(): +def function(depends_on, produces): # noqa: ARG001 ... @@ -348,8 +385,16 @@ def test_print_collected_tasks_with_nodes(capsys): base_name="function", path=Path("task_path.py"), function=function, - depends_on={0: MetaNode("in.txt")}, - produces={0: MetaNode("out.txt")}, + depends_on={ + "depends_on": FilePathNode( + name="in.txt", value=Path("in.txt"), path=Path("in.txt") + ) + }, + produces={ + 0: FilePathNode( + name="out.txt", value=Path("out.txt"), path=Path("out.txt") + ) + }, ) ] } @@ -372,9 +417,13 @@ def test_find_common_ancestor_of_all_nodes(show_nodes, expected_add): base_name="function", path=Path.cwd() / "src" / "task_path.py", function=function, - depends_on={0: MetaNode(Path.cwd() / "src" / "in.txt")}, + depends_on={ + "depends_on": FilePathNode.from_path(Path.cwd() / "src" / "in.txt") + }, produces={ - 0: MetaNode(Path.cwd().joinpath("..", "bld", "out.txt").resolve()) + 0: FilePathNode.from_path( + Path.cwd().joinpath("..", "bld", "out.txt").resolve() + ) }, ) ] From e832809eb3c097ef66186b9501d587045424a666 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Wed, 28 Jun 2023 23:25:58 +0200 Subject: [PATCH 4/7] Test clean command with new tasks. --- tests/test_clean.py | 66 ++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/tests/test_clean.py b/tests/test_clean.py index 4c4c0bdd6..62a7e6220 100644 --- a/tests/test_clean.py +++ b/tests/test_clean.py @@ -11,18 +11,29 @@ from pytask import ExitCode -@pytest.fixture() -def project(tmp_path): - """Create a sample project to be cleaned.""" - source = """ - import pytask +_PROJECT_TASK = """ +import pytask - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") - def task_write_text(depends_on, produces): - produces.write_text("a") - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) +@pytask.mark.depends_on("in.txt") +@pytask.mark.produces("out.txt") +def task_write_text(depends_on, produces): + produces.write_text("a") +""" + + +_PROJECT_TASK_NEW_INTERFACE = """ +import pytask +from pathlib import Path + +def task_write_text(in_path=Path("in.txt"), produces=Path("out.txt")): + produces.write_text("a") +""" + + +@pytest.fixture(params=[_PROJECT_TASK, _PROJECT_TASK_NEW_INTERFACE]) +def project(request, tmp_path): + """Create a sample project to be cleaned.""" + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(request.param)) tmp_path.joinpath("in.txt").touch() tmp_path.joinpath("to_be_deleted_file_1.txt").touch() @@ -32,18 +43,29 @@ def task_write_text(depends_on, produces): return tmp_path -@pytest.fixture() -def git_project(tmp_path): - """Create a sample project to be cleaned.""" - source = """ - import pytask +_GIT_PROJECT_TASK = """ +import pytask - @pytask.mark.depends_on("in_tracked.txt") - @pytask.mark.produces("out.txt") - def task_write_text(depends_on, produces): - produces.write_text("a") - """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) +@pytask.mark.depends_on("in_tracked.txt") +@pytask.mark.produces("out.txt") +def task_write_text(depends_on, produces): + produces.write_text("a") +""" + + +_GIT_PROJECT_TASK_NEW_INTERFACE = """ +import pytask +from pathlib import Path + +def task_write_text(in_path=Path("in_tracked.txt"), produces=Path("out.txt")): + produces.write_text("a") +""" + + +@pytest.fixture(params=[_GIT_PROJECT_TASK, _GIT_PROJECT_TASK_NEW_INTERFACE]) +def git_project(request, tmp_path): + """Create a sample project to be cleaned.""" + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(request.param)) tmp_path.joinpath("in_tracked.txt").touch() tmp_path.joinpath("tracked.txt").touch() From 5674f06c418696331261a0f5fc1e4cf5b7fc4115 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 30 Jun 2023 09:12:46 +0200 Subject: [PATCH 5/7] Parse produces from function defaults if decorator is not used. --- src/_pytask/collect.py | 6 ++- src/_pytask/collect_command.py | 6 +-- src/_pytask/collect_utils.py | 77 +++++++++++++++++++++------------- tests/test_collect.py | 24 ++++++++++- tests/test_collect_command.py | 3 +- 5 files changed, 77 insertions(+), 39 deletions(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 08d24c493..8403a9e8d 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -14,6 +14,7 @@ from _pytask.collect_utils import depends_on from _pytask.collect_utils import parse_dependencies_from_task_function from _pytask.collect_utils import parse_nodes +from _pytask.collect_utils import parse_products_from_task_function from _pytask.collect_utils import produces from _pytask.config import hookimpl from _pytask.config import IS_FILE_SYSTEM_CASE_SENSITIVE @@ -177,7 +178,10 @@ def pytask_collect_task( session, path, name, obj ) - products = parse_nodes(session, path, name, obj, produces) + if has_mark(obj, "produces"): + products = parse_nodes(session, path, name, obj, produces) + else: + products = parse_products_from_task_function(session, path, name, obj) markers = obj.pytask_meta.markers if hasattr(obj, "pytask_meta") else [] diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 977def55d..e297c823a 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -129,11 +129,7 @@ def _find_common_ancestor_of_all_nodes( for x in tree_just_flatten(task.depends_on) if isinstance(x, FilePathNode) ) - all_paths.extend( - x.path - for x in tree_just_flatten(task.produces) - if isinstance(x, FilePathNode) - ) + all_paths.extend(x.path for x in tree_just_flatten(task.produces)) common_ancestor = find_common_ancestor(*all_paths, *paths) diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 780bb8af8..e27fc84df 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -1,6 +1,7 @@ """This module provides utility functions for :mod:`_pytask.collect`.""" from __future__ import annotations +import inspect import itertools import uuid from pathlib import Path @@ -206,21 +207,6 @@ def _collect_node( ) -> dict[str, MetaNode]: """Collect nodes for a task. - Parameters - ---------- - session - The session. - path - The path to the task whose nodes are collected. - name - The name of the task. - nodes - A dictionary of nodes parsed from the ``depends_on`` or ``produces`` markers. - - Returns - ------- - A dictionary of node names and their paths. - Raises ------ NodeNotCollectedError @@ -229,7 +215,7 @@ def _collect_node( """ if not isinstance(node, (str, Path)): raise ValueError( - "'@pytask.mark.depends_on' and '@pytask.mark.produces' can only accept" + "'@pytask.mark.depends_on' and '@pytask.mark.produces' can only accept " "values of type 'str' and 'pathlib.Path' or the same values nested in " f"tuples, lists, and dictionaries. Here, {node} has type {type(node)}." ) @@ -258,26 +244,57 @@ def parse_dependencies_from_task_function( kwargs = {**signature_defaults, **task_kwargs} kwargs.pop("produces", None) - def _collect_node( - session: Session, path: Path, name: str, node: Any - ) -> dict[str, MetaNode]: - collected_node = session.hook.pytask_collect_node( - session=session, path=path, node=node - ) - if collected_node is None: - raise NodeNotCollectedError( - f"{node!r} cannot be parsed as a dependency for task " - f"{name!r} in {path!r}." - ) - return collected_node - dependencies = {} for name, value in kwargs.items(): parsed_value = tree_map( - lambda x: _collect_node(session, path, name, x), value # noqa: B023 + lambda x: _collect_new_node(session, path, name, x), value # noqa: B023 ) dependencies[name] = ( PythonNode(value=None) if parsed_value is None else parsed_value ) return dependencies + + +def parse_products_from_task_function( + session: Session, path: Path, name: str, obj: Any +) -> dict[str, Any]: + """Parse dependencies from task function.""" + task_kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {} + if "produces" in task_kwargs: + return tree_map( + lambda x: _collect_new_node(session, path, name, x), + task_kwargs["produces"], + ) + + parameters = inspect.signature(obj).parameters + if "produces" in parameters: + parameter = parameters["produces"] + if parameter.default is not parameter.empty: + return tree_map( + lambda x: _collect_new_node(session, path, name, x), + parameter.default, + ) + return {} + + +def _collect_new_node( + session: Session, path: Path, name: str, node: Any +) -> dict[str, MetaNode]: + """Collect nodes for a task. + + Raises + ------ + NodeNotCollectedError + If the node could not collected. + + """ + collected_node = session.hook.pytask_collect_node( + session=session, path=path, node=node + ) + if collected_node is None: + raise NodeNotCollectedError( + f"{node!r} cannot be parsed as a dependency for task " + f"{name!r} in {path!r}." + ) + return collected_node diff --git a/tests/test_collect.py b/tests/test_collect.py index 1766765e1..a9fb0fc1e 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -36,7 +36,7 @@ def task_write_text(depends_on, produces): @pytest.mark.end_to_end() -def test_collect_filepathnode_with_unknown_type(tmp_path): +def test_collect_depends_on_that_is_not_str_or_path(tmp_path): """If a node cannot be parsed because unknown type, raise an error.""" source = """ import pytask @@ -53,6 +53,28 @@ def task_with_non_path_dependency(): assert session.collection_reports[0].outcome == CollectionOutcome.FAIL exc_info = session.collection_reports[0].exc_info assert isinstance(exc_info[1], ValueError) + assert "'@pytask.mark.depends_on'" in str(exc_info[1]) + + +@pytest.mark.end_to_end() +def test_collect_produces_that_is_not_str_or_path(tmp_path): + """If a node cannot be parsed because unknown type, raise an error.""" + source = """ + import pytask + + @pytask.mark.produces(True) + def task_with_non_path_dependency(): + pass + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + session = main({"paths": tmp_path}) + + assert session.exit_code == ExitCode.COLLECTION_FAILED + assert session.collection_reports[0].outcome == CollectionOutcome.FAIL + exc_info = session.collection_reports[0].exc_info + assert isinstance(exc_info[1], ValueError) + assert "'@pytask.mark.depends_on'" in str(exc_info[1]) @pytest.mark.end_to_end() diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py index bfc23d927..74e0ba847 100644 --- a/tests/test_collect_command.py +++ b/tests/test_collect_command.py @@ -56,8 +56,6 @@ def test_collect_task_new_interface(runner, tmp_path): source = """ import pytask - @pytask.mark.depends_on("in.txt") - @pytask.mark.produces("out.txt") def task_example(depends_on="in.txt", arg=1, produces="out.txt"): pass """ @@ -85,6 +83,7 @@ def task_example(depends_on="in.txt", arg=1, produces="out.txt"): assert "in.txt>" in captured assert "" in captured + assert "arg" in captured @pytest.mark.end_to_end() From aa991f6eb2fe46d629de9adf3096ee1f90be77be Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 30 Jun 2023 20:54:04 +0200 Subject: [PATCH 6/7] Parse products correctly. --- src/_pytask/collect_utils.py | 121 ++++++++++++++++++++++++---------- tests/test_collect.py | 15 +++++ tests/test_collect_command.py | 4 +- 3 files changed, 104 insertions(+), 36 deletions(-) diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index e27fc84df..76251a820 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -202,39 +202,6 @@ def _merge_dictionaries(list_of_dicts: list[dict[Any, Any]]) -> dict[Any, Any]: return out -def _collect_node( - session: Session, path: Path, name: str, node: str | Path -) -> dict[str, MetaNode]: - """Collect nodes for a task. - - Raises - ------ - NodeNotCollectedError - If the node could not collected. - - """ - if not isinstance(node, (str, Path)): - raise ValueError( - "'@pytask.mark.depends_on' and '@pytask.mark.produces' can only accept " - "values of type 'str' and 'pathlib.Path' or the same values nested in " - f"tuples, lists, and dictionaries. Here, {node} has type {type(node)}." - ) - - if isinstance(node, str): - node = Path(node) - - collected_node = session.hook.pytask_collect_node( - session=session, path=path, node=node - ) - if collected_node is None: - raise NodeNotCollectedError( - f"{node!r} cannot be parsed as a dependency or product for task " - f"{name!r} in {path!r}." - ) - - return collected_node - - def parse_dependencies_from_task_function( session: Session, path: Path, name: str, obj: Any ) -> dict[str, Any]: @@ -263,7 +230,7 @@ def parse_products_from_task_function( task_kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {} if "produces" in task_kwargs: return tree_map( - lambda x: _collect_new_node(session, path, name, x), + lambda x: _collect_product(session, path, name, x, is_string_allowed=True), task_kwargs["produces"], ) @@ -271,13 +238,49 @@ def parse_products_from_task_function( if "produces" in parameters: parameter = parameters["produces"] if parameter.default is not parameter.empty: + # Use _collect_new_node to not collect strings. return tree_map( - lambda x: _collect_new_node(session, path, name, x), + lambda x: _collect_product( + session, path, name, x, is_string_allowed=False + ), parameter.default, ) return {} +def _collect_node( + session: Session, path: Path, name: str, node: str | Path +) -> dict[str, MetaNode]: + """Collect nodes for a task. + + Raises + ------ + NodeNotCollectedError + If the node could not collected. + + """ + if not isinstance(node, (str, Path)): + raise ValueError( + "'@pytask.mark.depends_on' and '@pytask.mark.produces' can only accept " + "values of type 'str' and 'pathlib.Path' or the same values nested in " + f"tuples, lists, and dictionaries. Here, {node} has type {type(node)}." + ) + + if isinstance(node, str): + node = Path(node) + + collected_node = session.hook.pytask_collect_node( + session=session, path=path, node=node + ) + if collected_node is None: + raise NodeNotCollectedError( + f"{node!r} cannot be parsed as a dependency or product for task " + f"{name!r} in {path!r}." + ) + + return collected_node + + def _collect_new_node( session: Session, path: Path, name: str, node: Any ) -> dict[str, MetaNode]: @@ -298,3 +301,51 @@ def _collect_new_node( f"{name!r} in {path!r}." ) return collected_node + + +def _collect_product( + session: Session, + path: Path, + name: str, + node: str | Path, + is_string_allowed: bool = False, +) -> dict[str, MetaNode]: + """Collect products for a task. + + Defining products with strings is only allowed when using the decorator. Parameter + defaults can only be :class:`pathlib.Path`s. + + Raises + ------ + NodeNotCollectedError + If the node could not collected. + + """ + # For historical reasons, task.kwargs is like the deco and supports str and Path. + if not isinstance(node, (str, Path)) and is_string_allowed: + raise ValueError( + "`@pytask.mark.task(kwargs={'produces': ...}` can only accept values of " + "type 'str' and 'pathlib.Path' or the same values nested in " + f"tuples, lists, and dictionaries. Here, {node} has type {type(node)}." + ) + # The parameter defaults only support Path objects. + if not isinstance(node, Path) and not is_string_allowed: + raise ValueError( + "If you use 'produces' as an argument of a task, it can only accept values " + "of type 'pathlib.Path' or the same value nested in " + f"tuples, lists, and dictionaries. Here, {node} has type {type(node)}." + ) + + if isinstance(node, str): + node = Path(node) + + collected_node = session.hook.pytask_collect_node( + session=session, path=path, node=node + ) + if collected_node is None: + raise NodeNotCollectedError( + f"{node!r} cannot be parsed as a dependency or product for task " + f"{name!r} in {path!r}." + ) + + return collected_node diff --git a/tests/test_collect.py b/tests/test_collect.py index a9fb0fc1e..95c2ef226 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -304,3 +304,18 @@ def task_my_task(): session = main({"paths": tmp_path}) outcome = session.collection_reports[0].outcome assert outcome == CollectionOutcome.SUCCESS + + +@pytest.mark.end_to_end() +def test_collect_string_product_as_function_default_fails(tmp_path): + source = """ + import pytask + + def task_write_text(produces="out.txt"): + produces.touch() + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + session = main({"paths": tmp_path}) + report = session.collection_reports[0] + assert report.outcome == CollectionOutcome.FAIL + assert "If you use 'produces'" in str(report.exc_info[1]) diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py index 74e0ba847..d7f3149e7 100644 --- a/tests/test_collect_command.py +++ b/tests/test_collect_command.py @@ -51,12 +51,14 @@ def task_example(): assert "out.txt>" in captured +@pytest.mark.xfail(reason="Only FilePathNodes are supported.") @pytest.mark.end_to_end() def test_collect_task_new_interface(runner, tmp_path): source = """ import pytask + from pathlib import Path - def task_example(depends_on="in.txt", arg=1, produces="out.txt"): + def task_example(depends_on=Path("in.txt"), arg=1, produces=Path("out.txt")): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) From 953a712a577574e73060885122c70ac83a30c3f6 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sat, 1 Jul 2023 10:46:20 +0200 Subject: [PATCH 7/7] Last touches. --- src/_pytask/collect_utils.py | 8 ++++---- src/_pytask/nodes.py | 3 ++- src/_pytask/report.py | 1 + tests/test_build.py | 1 - 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 76251a820..5eb6cff9a 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -72,7 +72,7 @@ def parse_nodes( """Parse nodes from object.""" objects = _extract_nodes_from_function_markers(obj, parser) nodes = _convert_objects_to_node_dictionary(objects, parser.__name__) - nodes = tree_map(lambda x: _collect_node(session, path, name, x), nodes) + nodes = tree_map(lambda x: _collect_old_dependencies(session, path, name, x), nodes) return nodes @@ -214,7 +214,7 @@ def parse_dependencies_from_task_function( dependencies = {} for name, value in kwargs.items(): parsed_value = tree_map( - lambda x: _collect_new_node(session, path, name, x), value # noqa: B023 + lambda x: _collect_dependencies(session, path, name, x), value # noqa: B023 ) dependencies[name] = ( PythonNode(value=None) if parsed_value is None else parsed_value @@ -248,7 +248,7 @@ def parse_products_from_task_function( return {} -def _collect_node( +def _collect_old_dependencies( session: Session, path: Path, name: str, node: str | Path ) -> dict[str, MetaNode]: """Collect nodes for a task. @@ -281,7 +281,7 @@ def _collect_node( return collected_node -def _collect_new_node( +def _collect_dependencies( session: Session, path: Path, name: str, node: Any ) -> dict[str, MetaNode]: """Collect nodes for a task. diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index 64db82849..47f7b9f19 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -117,7 +117,8 @@ class PythonNode(MetaNode): name: str = "" def __attrs_post_init__(self) -> None: - self.name = str(self.value) + if not self.name: + self.name = str(self.value) def state(self) -> str | None: if self.hash: diff --git a/src/_pytask/report.py b/src/_pytask/report.py index ad9abe30d..0bc0cf03d 100644 --- a/src/_pytask/report.py +++ b/src/_pytask/report.py @@ -37,6 +37,7 @@ def from_exception( exc_info: ExceptionInfo, node: MetaNode | None = None, ) -> CollectionReport: + exc_info = remove_internal_traceback_frames_from_exc_info(exc_info) return cls(outcome=outcome, node=node, exc_info=exc_info) diff --git a/tests/test_build.py b/tests/test_build.py index 75d1d4e8d..96f158e6f 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -38,7 +38,6 @@ def test_collection_failed(runner, tmp_path): @pytest.mark.end_to_end() def test_building_dag_failed(runner, tmp_path): - # TODO: Should maybe fail because depends_on and produces are not used. source = """ import pytask