From d9e21c193114e75007fadffda97d9f6b986fed44 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Tue, 29 Aug 2023 19:13:31 +0200 Subject: [PATCH 01/25] Rename main function to build. --- scripts/svgs/task_dry_run.py | 2 +- scripts/svgs/task_hello.py | 2 +- scripts/svgs/task_write_task.py | 2 +- src/_pytask/build.py | 10 +++++----- src/_pytask/collect_utils.py | 2 +- src/pytask/__init__.py | 4 ++-- tests/test_collect.py | 24 ++++++++++++------------ tests/test_config.py | 8 ++++---- tests/test_debugging.py | 2 +- tests/test_execute.py | 26 +++++++++++++------------- tests/test_ignore.py | 6 +++--- tests/test_mark.py | 10 +++++----- tests/test_mark_cli.py | 4 ++-- tests/test_persist.py | 12 ++++++------ tests/test_profile.py | 4 ++-- tests/test_shared.py | 4 ++-- tests/test_skipping.py | 22 +++++++++++----------- tests/test_task.py | 6 +++--- tests/test_tree_util.py | 4 ++-- tests/test_warnings.py | 4 ++-- 20 files changed, 79 insertions(+), 79 deletions(-) diff --git a/scripts/svgs/task_dry_run.py b/scripts/svgs/task_dry_run.py index 1e9e3a602..dda7abe0b 100644 --- a/scripts/svgs/task_dry_run.py +++ b/scripts/svgs/task_dry_run.py @@ -12,5 +12,5 @@ def task_dry_run(produces: Path) -> None: if __name__ == "__main__": pytask.console.record = True - pytask.main({"paths": __file__, "dry_run": True}) + pytask.build({"paths": __file__, "dry_run": True}) pytask.console.save_svg("dry-run.svg", title="pytask") diff --git a/scripts/svgs/task_hello.py b/scripts/svgs/task_hello.py index 7e35cb2cc..9e0dd6851 100644 --- a/scripts/svgs/task_hello.py +++ b/scripts/svgs/task_hello.py @@ -12,5 +12,5 @@ def task_hello_earth(produces: Path) -> None: if __name__ == "__main__": pytask.console.record = True - pytask.main({"paths": __file__}) + pytask.build({"paths": __file__}) pytask.console.save_svg("readme.svg", title="pytask") diff --git a/scripts/svgs/task_write_task.py b/scripts/svgs/task_write_task.py index ab60d2246..ef2f8d534 100644 --- a/scripts/svgs/task_write_task.py +++ b/scripts/svgs/task_write_task.py @@ -24,7 +24,7 @@ def task_create_random_data(produces: Path) -> None: if __name__ == "__main__": pytask.console.record = True - pytask.main({"paths": __file__}) + pytask.build({"paths": __file__}) pytask.console.save_svg("write-a-task.svg", title="pytask") Path(__file__).parent.joinpath("data.pkl").unlink() diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 7585f5822..6fdbb2e38 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -32,10 +32,10 @@ @hookimpl(tryfirst=True) def pytask_extend_command_line_interface(cli: click.Group) -> None: """Extend the command line interface.""" - cli.add_command(build) + cli.add_command(build_command) -def main(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 +def build(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 """Run pytask. This is the main command to run pytask which usually receives kwargs from the @@ -137,7 +137,7 @@ def main(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 return session -@click.command(cls=ColoredCommand) +@click.command(cls=ColoredCommand, name="build") @click.option( "--debug-pytask", is_flag=True, @@ -179,7 +179,7 @@ def main(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 default=False, help="Execute a task even if it succeeded successfully before.", ) -def build(**raw_config: Any) -> NoReturn: +def build_command(**raw_config: Any) -> NoReturn: """Collect tasks, execute them and report the results. The default command. pytask collects tasks from the given paths or the @@ -187,5 +187,5 @@ def build(**raw_config: Any) -> NoReturn: """ raw_config["command"] = "build" - session = main(raw_config) + session = build(raw_config) sys.exit(session.exit_code) diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index d10549dc3..5d0bf273c 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -285,7 +285,7 @@ def parse_dependencies_from_task_function( ) # If all nodes are python nodes, we simplify the parameter value and store it in - # one node. + # one node. If it is a node, we keep it. are_all_nodes_python_nodes_without_hash = all( isinstance(x, PythonNode) and not x.hash for x in tree_leaves(nodes) ) diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py index d626d4dd3..f71c15d98 100644 --- a/src/pytask/__init__.py +++ b/src/pytask/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations from _pytask import __version__ -from _pytask.build import main +from _pytask.build import build from _pytask.click import ColoredCommand from _pytask.click import ColoredGroup from _pytask.click import EnumChoice @@ -120,7 +120,7 @@ "has_mark", "hookimpl", "import_optional_dependency", - "main", + "build", "mark", "parse_nodes", "parse_warning_filter", diff --git a/tests/test_collect.py b/tests/test_collect.py index 0c56db3c8..c83939283 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -10,10 +10,10 @@ from _pytask.collect import pytask_collect_node from _pytask.exceptions import NodeNotCollectedError from _pytask.models import NodeInfo +from pytask import build from pytask import cli from pytask import CollectionOutcome from pytask import ExitCode -from pytask import main from pytask import Session from pytask import Task @@ -31,7 +31,7 @@ def task_write_text(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Relative paths work.") - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.collection_reports[0].outcome == CollectionOutcome.SUCCESS assert tmp_path.joinpath("out.txt").read_text() == "Relative paths work." @@ -49,7 +49,7 @@ def task_with_non_path_dependency(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.COLLECTION_FAILED assert session.collection_reports[0].outcome == CollectionOutcome.FAIL @@ -70,7 +70,7 @@ def task_with_non_path_dependency(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.COLLECTION_FAILED assert session.collection_reports[0].outcome == CollectionOutcome.FAIL @@ -114,7 +114,7 @@ def task_1(depends_on, produces): def test_collect_same_task_different_ways(tmp_path, path_extension): tmp_path.joinpath("task_module.py").write_text("def task_passes(): pass") - session = main({"paths": tmp_path.joinpath(path_extension)}) + session = build({"paths": tmp_path.joinpath(path_extension)}) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 @@ -141,7 +141,7 @@ def test_collect_files_w_custom_file_name_pattern( for file in task_files: tmp_path.joinpath(file).write_text("def task_example(): pass") - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert len(session.tasks) == expected_collected_tasks @@ -264,7 +264,7 @@ def task_example(path_in = Path("in.txt"), produces = Path("out.txt")): tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("hello") - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 @@ -278,7 +278,7 @@ def test_collect_tasks_from_modules_with_the_same_name(tmp_path): tmp_path.joinpath("b").mkdir() tmp_path.joinpath("a", "task_module.py").write_text("def task_a(): pass") tmp_path.joinpath("b", "task_module.py").write_text("def task_a(): pass") - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert len(session.collection_reports) == 2 assert all( report.outcome == CollectionOutcome.SUCCESS @@ -306,7 +306,7 @@ def task_my_task(): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) outcome = session.collection_reports[0].outcome assert outcome == CollectionOutcome.SUCCESS @@ -368,7 +368,7 @@ def task_example( ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.COLLECTION_FAILED assert len(session.tasks) == 0 @@ -395,7 +395,7 @@ def task_example( tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("input_1.txt").touch() tmp_path.joinpath("input_2.txt").touch() - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.COLLECTION_FAILED assert len(session.tasks) == 0 @@ -439,7 +439,7 @@ def task_example( """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": [tmp_path]}) + session = build({"paths": [tmp_path]}) assert session.exit_code == ExitCode.OK product = session.tasks[0].produces["path"] assert product.name == "product" diff --git a/tests/test_config.py b/tests/test_config.py index 130f73b1a..320f078ce 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3,13 +3,13 @@ import textwrap import pytest +from pytask import build from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() def test_debug_pytask(capsys, tmp_path): - session = main({"paths": tmp_path, "debug_pytask": True}) + session = build({"paths": tmp_path, "debug_pytask": True}) assert session.exit_code == ExitCode.OK @@ -31,7 +31,7 @@ def test_pass_config_to_cli(tmp_path): """ tmp_path.joinpath("pyproject.toml").write_text(textwrap.dedent(config)) - session = main({"config": tmp_path.joinpath("pyproject.toml"), "paths": tmp_path}) + session = build({"config": tmp_path.joinpath("pyproject.toml"), "paths": tmp_path}) assert session.exit_code == ExitCode.OK assert "elton" in session.config["markers"] @@ -55,7 +55,7 @@ def test_passing_paths_via_configuration_file(tmp_path, file_or_folder): "def task_passes(): pass" ) - session = main({"config": tmp_path.joinpath("pyproject.toml")}) + session = build({"config": tmp_path.joinpath("pyproject.toml")}) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 diff --git a/tests/test_debugging.py b/tests/test_debugging.py index 6687bd0d4..5502bac49 100644 --- a/tests/test_debugging.py +++ b/tests/test_debugging.py @@ -473,7 +473,7 @@ def test_set_trace_is_returned_after_pytask_finishes(tmp_path): import pytask def test_function(): - pytask.main({{"paths": "{tmp_path.as_posix()}"}}) + pytask.build({{"paths": "{tmp_path.as_posix()}"}}) breakpoint() """ tmp_path.joinpath("test_example.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_execute.py b/tests/test_execute.py index 442c471a9..8ce120e25 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -11,9 +11,9 @@ import pytest from _pytask.capture import _CaptureMethod from _pytask.exceptions import NodeNotFoundError +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main from pytask import TaskOutcome @@ -46,7 +46,7 @@ def task_example(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.FAILED assert len(session.execution_reports) == 1 @@ -103,7 +103,7 @@ def task_3(depends_on): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.FAILED assert sum(i.outcome == TaskOutcome.SUCCESS for i in session.execution_reports) == 2 @@ -138,7 +138,7 @@ def task_example(depends_on, produces): for dependency in dependencies: tmp_path.joinpath(dependency).touch() - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK @@ -157,7 +157,7 @@ def task_example(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Here I am. Once again.") - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert tmp_path.joinpath("out.txt").read_text() == "Here I am. Once again." @@ -243,7 +243,7 @@ def task_example(depends_on, produces): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK @@ -257,7 +257,7 @@ def task_3(): raise Exception """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path, "max_failures": n_failures}) + session = build({"paths": tmp_path, "max_failures": n_failures}) assert len(session.tasks) == 3 assert len(session.execution_reports) == n_failures @@ -273,7 +273,7 @@ def task_3(): raise Exception """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main( + session = build( {"paths": tmp_path, "stop_after_first_failure": stop_after_first_failure} ) @@ -296,7 +296,7 @@ def task_y(): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert session.execution_reports[0].task.name.endswith("task_z") @@ -390,9 +390,9 @@ def task_example(): @pytest.mark.end_to_end() def test_task_executed_with_force_although_unchanged(tmp_path): tmp_path.joinpath("task_module.py").write_text("def task_example(): pass") - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS - session = main({"paths": tmp_path, "force": True}) + session = build({"paths": tmp_path, "force": True}) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS @@ -436,7 +436,7 @@ def task_example(path_to_file: Annotated[Path, Product] = Path("out.txt")) -> No """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path, "capture": _CaptureMethod.NO}) + session = build({"paths": tmp_path, "capture": _CaptureMethod.NO}) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 @@ -459,7 +459,7 @@ def task_example( """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path, "capture": _CaptureMethod.NO}) + session = build({"paths": tmp_path, "capture": _CaptureMethod.NO}) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 diff --git a/tests/test_ignore.py b/tests/test_ignore.py index 7c95483d9..bf76f54bf 100644 --- a/tests/test_ignore.py +++ b/tests/test_ignore.py @@ -5,8 +5,8 @@ import pytest from _pytask.collect import pytask_ignore_collect from _pytask.config import _IGNORED_FOLDERS +from pytask import build from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() @@ -16,7 +16,7 @@ def test_ignore_default_paths(tmp_path, ignored_folder): tmp_path.joinpath(folder).mkdir() tmp_path.joinpath(folder, "task_module.py").write_text("def task_d(): pass") - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 0 @@ -32,7 +32,7 @@ def test_ignore_paths(tmp_path, ignore, new_line): ) tmp_path.joinpath("pyproject.toml").write_text(config) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 0 if ignore else len(session.tasks) == 1 diff --git a/tests/test_mark.py b/tests/test_mark.py index a1fb7773b..82f05a1d0 100644 --- a/tests/test_mark.py +++ b/tests/test_mark.py @@ -6,9 +6,9 @@ import pytask import pytest +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main from pytask import MarkGenerator @@ -109,7 +109,7 @@ def task_two(): """ ) ) - session = main({"paths": tmp_path, "marker_expression": expr}) + session = build({"paths": tmp_path, "marker_expression": expr}) tasks_that_run = [ report.task.name.rsplit("::")[1] @@ -152,7 +152,7 @@ def task_no_2(): """ ) ) - session = main({"paths": tmp_path, "expression": expr}) + session = build({"paths": tmp_path, "expression": expr}) assert session.exit_code == ExitCode.OK tasks_that_run = [ @@ -184,7 +184,7 @@ def task_func(arg=arg): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path, "expression": expr}) + session = build({"paths": tmp_path, "expression": expr}) assert session.exit_code == ExitCode.OK tasks_that_run = [ @@ -233,7 +233,7 @@ def test_keyword_option_wrong_arguments( tmp_path.joinpath("task_module.py").write_text( textwrap.dedent("def task_func(arg): pass") ) - session = main({"paths": tmp_path, option: expr}) + session = build({"paths": tmp_path, option: expr}) assert session.exit_code == ExitCode.DAG_FAILED captured = capsys.readouterr() diff --git a/tests/test_mark_cli.py b/tests/test_mark_cli.py index c1754f27c..2a6f5ce77 100644 --- a/tests/test_mark_cli.py +++ b/tests/test_mark_cli.py @@ -3,9 +3,9 @@ import textwrap import pytest +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() @@ -48,5 +48,5 @@ def test_marker_names(tmp_path, marker_name): markers = ['{marker_name}'] """ tmp_path.joinpath("pyproject.toml").write_text(textwrap.dedent(toml)) - session = main({"paths": tmp_path, "markers": True}) + session = build({"paths": tmp_path, "markers": True}) assert session.exit_code == ExitCode.CONFIGURATION_FAILED diff --git a/tests/test_persist.py b/tests/test_persist.py index 071155841..668e11901 100644 --- a/tests/test_persist.py +++ b/tests/test_persist.py @@ -5,10 +5,10 @@ import pytest from _pytask.database_utils import State from _pytask.persist import pytask_execute_task_process_report +from pytask import build from pytask import create_database from pytask import DatabaseSession from pytask import ExitCode -from pytask import main from pytask import Persisted from pytask import SkippedUnchanged from pytask import TaskOutcome @@ -20,7 +20,7 @@ class DummyClass: @pytest.mark.end_to_end() def test_persist_marker_is_set(tmp_path): - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert "persist" in session.config["markers"] @@ -46,7 +46,7 @@ def task_dummy(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("I'm not the reason you care.") - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 @@ -56,7 +56,7 @@ def task_dummy(depends_on, produces): tmp_path.joinpath("out.txt").write_text("Never again in despair.") - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 @@ -72,7 +72,7 @@ def task_dummy(depends_on, produces): modification_time = session.get(State, (task_id, node_id)).modification_time assert float(modification_time) == tmp_path.joinpath("out.txt").stat().st_mtime - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 @@ -97,7 +97,7 @@ def task_dummy(depends_on, produces): "They say oh my god I see the way you shine." ) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 diff --git a/tests/test_profile.py b/tests/test_profile.py index fe241aa54..b3ba99a21 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -8,10 +8,10 @@ from _pytask.cli import cli from _pytask.profile import _to_human_readable_size from _pytask.profile import Runtime +from pytask import build from pytask import create_database from pytask import DatabaseSession from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() @@ -22,7 +22,7 @@ def task_example(): time.sleep(2) """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 diff --git a/tests/test_shared.py b/tests/test_shared.py index 9383b79c1..b994d20b8 100644 --- a/tests/test_shared.py +++ b/tests/test_shared.py @@ -5,7 +5,7 @@ import pytest from _pytask.outcomes import ExitCode from _pytask.shared import find_duplicates -from pytask import main +from pytask import build @pytest.mark.unit() @@ -27,7 +27,7 @@ def test_parse_markers(tmp_path): """ tmp_path.joinpath("pyproject.toml").write_text(textwrap.dedent(toml)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK assert "a1" in session.config["markers"] diff --git a/tests/test_skipping.py b/tests/test_skipping.py index bb18d0fae..a825f0157 100644 --- a/tests/test_skipping.py +++ b/tests/test_skipping.py @@ -6,9 +6,9 @@ import pytest from _pytask.skipping import pytask_execute_task_setup +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main from pytask import Mark from pytask import Session from pytask import Skipped @@ -30,10 +30,10 @@ def task_dummy(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert isinstance(session.execution_reports[0].exc_info[1], SkippedUnchanged) @@ -50,12 +50,12 @@ def task_dummy(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Original content of in.txt.") - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS assert tmp_path.joinpath("out.txt").read_text() == "Original content of in.txt." - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.execution_reports[0].outcome == TaskOutcome.SKIP_UNCHANGED assert isinstance(session.execution_reports[0].exc_info[1], SkippedUnchanged) @@ -77,7 +77,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.execution_reports[0].outcome == TaskOutcome.FAIL assert isinstance(session.execution_reports[0].exc_info[1], Exception) @@ -101,7 +101,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.execution_reports[0].outcome == TaskOutcome.SKIP assert isinstance(session.execution_reports[0].exc_info[1], Skipped) @@ -124,7 +124,7 @@ def task_first(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.execution_reports[0].outcome == TaskOutcome.SKIP assert isinstance(session.execution_reports[0].exc_info[1], Skipped) @@ -173,7 +173,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) node = session.collection_reports[0].node assert len(node.markers) == 1 assert node.markers[0].name == "skipif" @@ -203,7 +203,7 @@ def task_second(depends_on): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) node = session.collection_reports[0].node assert len(node.markers) == 1 @@ -234,7 +234,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) node = session.collection_reports[0].node assert len(node.markers) == 2 assert node.markers[0].name == "skipif" diff --git a/tests/test_task.py b/tests/test_task.py index 7bf9157b1..fe978edff 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -3,9 +3,9 @@ import textwrap import pytest +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() @@ -23,7 +23,7 @@ def {func_name}(produces): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.OK @@ -434,7 +434,7 @@ def task_func(i=i): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == ExitCode.COLLECTION_FAILED assert isinstance(session.collection_reports[0].exc_info[1], ValueError) diff --git a/tests/test_tree_util.py b/tests/test_tree_util.py index 2f39b1a74..355844b04 100644 --- a/tests/test_tree_util.py +++ b/tests/test_tree_util.py @@ -7,8 +7,8 @@ from _pytask.outcomes import ExitCode from _pytask.tree_util import tree_map from _pytask.tree_util import tree_structure +from pytask import build from pytask import cli -from pytask import main @pytest.mark.end_to_end() @@ -40,7 +40,7 @@ def task_example(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path}) + session = build({"paths": tmp_path}) assert session.exit_code == exit_code diff --git a/tests/test_warnings.py b/tests/test_warnings.py index c5b2e763d..4d973d31b 100644 --- a/tests/test_warnings.py +++ b/tests/test_warnings.py @@ -5,9 +5,9 @@ import textwrap import pytest +from pytask import build from pytask import cli from pytask import ExitCode -from pytask import main @pytest.mark.end_to_end() @@ -46,7 +46,7 @@ def task_example(): """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) - session = main({"paths": tmp_path, "disable_warnings": disable_warnings}) + session = build({"paths": tmp_path, "disable_warnings": disable_warnings}) assert session.exit_code == ExitCode.OK if disable_warnings: From 661b789221d11c98ca4c79ae31cccdf600ee7a86 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Tue, 29 Aug 2023 23:16:07 +0200 Subject: [PATCH 02/25] Adding all base args to build function. --- .../how_to_guides/invoking_pytask_extended.md | 2 +- docs/source/reference_guides/api.md | 2 +- docs/source/tutorials/visualizing_the_dag.md | 2 +- scripts/svgs/task_dry_run.py | 2 +- scripts/svgs/task_hello.py | 2 +- scripts/svgs/task_write_task.py | 2 +- src/_pytask/build.py | 77 +++++++++++++++++-- src/_pytask/capture.py | 20 ++--- src/_pytask/clean.py | 2 +- src/_pytask/collect_command.py | 2 +- src/_pytask/mark/__init__.py | 2 +- src/_pytask/parameters.py | 4 +- src/_pytask/profile.py | 2 +- tests/test_capture.py | 12 +-- tests/test_collect.py | 22 +++--- tests/test_config.py | 6 +- tests/test_execute.py | 28 ++++--- tests/test_ignore.py | 4 +- tests/test_mark.py | 8 +- tests/test_mark_cli.py | 2 +- tests/test_persist.py | 10 +-- tests/test_profile.py | 2 +- tests/test_shared.py | 2 +- tests/test_skipping.py | 20 ++--- tests/test_task.py | 4 +- tests/test_tree_util.py | 2 +- tests/test_warnings.py | 2 +- 27 files changed, 153 insertions(+), 92 deletions(-) diff --git a/docs/source/how_to_guides/invoking_pytask_extended.md b/docs/source/how_to_guides/invoking_pytask_extended.md index 8f154a803..75be59180 100644 --- a/docs/source/how_to_guides/invoking_pytask_extended.md +++ b/docs/source/how_to_guides/invoking_pytask_extended.md @@ -10,7 +10,7 @@ Invoke pytask programmatically with import pytask -session = pytask.main({"paths": ...}) +session = pytask.build(paths=...) ``` Pass command line arguments with their long name and hyphens replaced by underscores as diff --git a/docs/source/reference_guides/api.md b/docs/source/reference_guides/api.md index bfd922e29..a39fe2801 100644 --- a/docs/source/reference_guides/api.md +++ b/docs/source/reference_guides/api.md @@ -325,7 +325,7 @@ outcome. ```{eval-rst} .. autofunction:: pytask.build_dag -.. autofunction:: pytask.main +.. autofunction:: pytask.build ``` ## Reports diff --git a/docs/source/tutorials/visualizing_the_dag.md b/docs/source/tutorials/visualizing_the_dag.md index 2930bed4c..3e365e416 100644 --- a/docs/source/tutorials/visualizing_the_dag.md +++ b/docs/source/tutorials/visualizing_the_dag.md @@ -37,7 +37,7 @@ layouts, which are listed [here](https://graphviz.org/docs/layouts/). The programmatic and interactive interface allows customizing the figure. -Similar to {func}`pytask.main`, there exists {func}`pytask.build_dag` which returns the +Similar to {func}`pytask.build`, there exists {func}`pytask.build_dag` which returns the DAG as a {class}`networkx.DiGraph`. ```python diff --git a/scripts/svgs/task_dry_run.py b/scripts/svgs/task_dry_run.py index dda7abe0b..623255071 100644 --- a/scripts/svgs/task_dry_run.py +++ b/scripts/svgs/task_dry_run.py @@ -12,5 +12,5 @@ def task_dry_run(produces: Path) -> None: if __name__ == "__main__": pytask.console.record = True - pytask.build({"paths": __file__, "dry_run": True}) + pytask.build(paths=__file__, dry_run=True) pytask.console.save_svg("dry-run.svg", title="pytask") diff --git a/scripts/svgs/task_hello.py b/scripts/svgs/task_hello.py index 9e0dd6851..da5ecfc77 100644 --- a/scripts/svgs/task_hello.py +++ b/scripts/svgs/task_hello.py @@ -12,5 +12,5 @@ def task_hello_earth(produces: Path) -> None: if __name__ == "__main__": pytask.console.record = True - pytask.build({"paths": __file__}) + pytask.build(paths=__file__) pytask.console.save_svg("readme.svg", title="pytask") diff --git a/scripts/svgs/task_write_task.py b/scripts/svgs/task_write_task.py index ef2f8d534..44d7c5715 100644 --- a/scripts/svgs/task_write_task.py +++ b/scripts/svgs/task_write_task.py @@ -24,7 +24,7 @@ def task_create_random_data(produces: Path) -> None: if __name__ == "__main__": pytask.console.record = True - pytask.build({"paths": __file__}) + pytask.build(paths=__file__) pytask.console.save_svg("write-a-task.svg", title="pytask") Path(__file__).parent.joinpath("data.pkl").unlink() diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 6fdbb2e38..392218e58 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -4,9 +4,11 @@ import sys from pathlib import Path from typing import Any +from typing import Iterable from typing import TYPE_CHECKING import click +from _pytask.capture import CaptureMethod from _pytask.click import ColoredCommand from _pytask.config import hookimpl from _pytask.config_utils import _find_project_root_and_config @@ -35,7 +37,37 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: cli.add_command(build_command) -def build(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR0915 +def build( # noqa: C901, PLR0912, PLR0913, PLR0915 + capture: CaptureMethod = CaptureMethod.NO, + check_casing_of_paths: bool = True, + config: Path | None = None, + database_url: str = "", + debug_pytask: bool = False, + disable_warnings: bool = False, + dry_run: bool = False, + editor_url_scheme: str = "file", + expression: str = "", + force: bool = False, + ignore: Iterable[str] = (), + marker_expression: str = "", + max_failures: float = float("inf"), + n_entries_in_table: int = 15, + paths: str | Path | Iterable[str | Path] = (), + pdb: bool = False, + pdb_cls: str = "", + s: bool = False, + show_capture: bool = True, + show_errors_immediately: bool = False, + show_locals: bool = False, + show_traceback: bool = True, + sort_table: bool = True, + stop_after_first_failure: bool = False, + strict_markers: bool = False, + task_files: str | Iterable[str] = "task_*.py", + trace: bool = False, + verbose: int = 1, + **kwargs: Any, +) -> Session: """Run pytask. This is the main command to run pytask which usually receives kwargs from the @@ -44,9 +76,8 @@ def build(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR091 Parameters ---------- - raw_config : dict[str, Any] - A dictionary with options passed to pytask. In general, this dictionary holds - the information passed via the command line interface. + capture + Defines the capture method. Returns ------- @@ -61,6 +92,38 @@ def build(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR091 pm.register(cli) pm.hook.pytask_add_hooks(pm=pm) + raw_config = { + "capture": capture, + "check_casing_of_paths": check_casing_of_paths, + "config": config, + "database_url": database_url, + "debug_pytask": debug_pytask, + "disable_warnings": disable_warnings, + "dry_run": dry_run, + "editor_url_scheme": editor_url_scheme, + "expression": expression, + "force": force, + "ignore": ignore, + "marker_expression": marker_expression, + "max_failures": max_failures, + "n_entries_in_table": n_entries_in_table, + "paths": paths, + "pdb": pdb, + "pdb_cls": pdb_cls, + "s": s, + "show_capture": show_capture, + "show_errors_immediately": show_errors_immediately, + "show_locals": show_locals, + "show_traceback": show_traceback, + "sort_table": sort_table, + "stop_after_first_failure": stop_after_first_failure, + "strict_markers": strict_markers, + "task_files": task_files, + "trace": trace, + "verbose": verbose, + **kwargs, + } + # If someone called the programmatic interface, we need to do some parsing. if "command" not in raw_config: raw_config["command"] = "build" @@ -97,9 +160,9 @@ def build(raw_config: dict[str, Any]) -> Session: # noqa: C901, PLR0912, PLR091 raw_config = {**raw_config, **config_from_file} - config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) + config_ = pm.hook.pytask_configure(pm=pm, raw_config=raw_config) - session = Session.from_config(config) + session = Session.from_config(config_) except (ConfigurationError, Exception): exc_info = sys.exc_info() @@ -187,5 +250,5 @@ def build_command(**raw_config: Any) -> NoReturn: """ raw_config["command"] = "build" - session = build(raw_config) + session = build(**raw_config) sys.exit(session.exit_code) diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index 777c76c79..501d84e84 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -47,7 +47,7 @@ from _pytask.nodes import Task -class _CaptureMethod(enum.Enum): +class CaptureMethod(enum.Enum): FD = "fd" NO = "no" SYS = "sys" @@ -60,8 +60,8 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: additional_parameters = [ click.Option( ["--capture"], - type=EnumChoice(_CaptureMethod), - default=_CaptureMethod.FD, + type=EnumChoice(CaptureMethod), + default=CaptureMethod.FD, help="Per task capturing method.", ), click.Option( @@ -88,7 +88,7 @@ def pytask_parse_config(config: dict[str, Any]) -> None: """ if config["s"]: - config["capture"] = _CaptureMethod.NO + config["capture"] = CaptureMethod.NO @hookimpl @@ -630,20 +630,20 @@ def readouterr(self) -> CaptureResult[AnyStr]: return CaptureResult(out, err) # type: ignore -def _get_multicapture(method: _CaptureMethod) -> MultiCapture[str]: +def _get_multicapture(method: CaptureMethod) -> MultiCapture[str]: """Set up the MultiCapture class with the passed method. For each valid method, the function instantiates the :class:`MultiCapture` class with the specified buffers for ``stdin``, ``stdout``, and ``stderr``. """ - if method == _CaptureMethod.FD: + if method == CaptureMethod.FD: return MultiCapture(in_=FDCapture(0), out=FDCapture(1), err=FDCapture(2)) - if method == _CaptureMethod.SYS: + if method == CaptureMethod.SYS: return MultiCapture(in_=SysCapture(0), out=SysCapture(1), err=SysCapture(2)) - if method == _CaptureMethod.NO: + if method == CaptureMethod.NO: return MultiCapture(in_=None, out=None, err=None) - if method == _CaptureMethod.TEE_SYS: + if method == CaptureMethod.TEE_SYS: return MultiCapture( in_=None, out=SysCapture(1, tee=True), err=SysCapture(2, tee=True) ) @@ -666,7 +666,7 @@ class CaptureManager: """ - def __init__(self, method: _CaptureMethod) -> None: + def __init__(self, method: CaptureMethod) -> None: self._method = method self._capturing: MultiCapture[str] | None = None diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index ecf515626..cb50af229 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -99,7 +99,7 @@ def clean(**raw_config: Any) -> NoReturn: # noqa: C901, PLR0912, PLR0915 raw_config["command"] = "clean" try: - # Duplication of the same mechanism in :func:`pytask.main.main`. + # Duplication of the same mechanism in :func:`pytask.build`. pm = get_plugin_manager() from _pytask import cli diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 39165beef..4a66b8e72 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -54,7 +54,7 @@ def collect(**raw_config: Any | None) -> NoReturn: raw_config["command"] = "collect" try: - # Duplication of the same mechanism in :func:`pytask.main.main`. + # Duplication of the same mechanism in :func:`pytask.build`. pm = get_plugin_manager() from _pytask import cli diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py index 83a603291..736b9c600 100644 --- a/src/_pytask/mark/__init__.py +++ b/src/_pytask/mark/__init__.py @@ -50,7 +50,7 @@ def markers(**raw_config: Any) -> NoReturn: raw_config["command"] = "markers" try: - # Duplication of the same mechanism in :func:`pytask.main.main`. + # Duplication of the same mechanism in :func:`pytask.build`. pm = get_plugin_manager() from _pytask import cli diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index 2ec31f53b..00d232e56 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -36,8 +36,8 @@ type=str, multiple=True, help=( - "A pattern to ignore files or directories. Refer to pathlib.Path.match for " - "more info." + "A pattern to ignore files or directories. Refer to :meth:`pathlib.Path.match` " + "for more info." ), default=[], ) diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 5c8a33e57..0ccbfbdf0 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -115,7 +115,7 @@ def profile(**raw_config: Any) -> NoReturn: raw_config["command"] = "profile" try: - # Duplication of the same mechanism in :func:`pytask.main.main`. + # Duplication of the same mechanism in :func:`pytask.build`. pm = get_plugin_manager() from _pytask import cli diff --git a/tests/test_capture.py b/tests/test_capture.py index 0cda066a8..b078ab9b0 100644 --- a/tests/test_capture.py +++ b/tests/test_capture.py @@ -13,9 +13,9 @@ import pytest from _pytask import capture -from _pytask.capture import _CaptureMethod from _pytask.capture import _get_multicapture from _pytask.capture import CaptureManager +from _pytask.capture import CaptureMethod from _pytask.capture import CaptureResult from _pytask.capture import MultiCapture from pytask import cli @@ -99,7 +99,7 @@ def TeeStdCapture( # noqa: N802 @pytest.mark.end_to_end() class TestCaptureManager: @pytest.mark.parametrize( - "method", [_CaptureMethod.NO, _CaptureMethod.SYS, _CaptureMethod.FD] + "method", [CaptureMethod.NO, CaptureMethod.SYS, CaptureMethod.FD] ) def test_capturing_basic_api(self, method): capouter = StdCaptureFD() @@ -116,7 +116,7 @@ def test_capturing_basic_api(self, method): print("hello") capman.suspend() out, err = capman.read() - if method == _CaptureMethod.NO: + if method == CaptureMethod.NO: assert old == (sys.stdout, sys.stderr, sys.stdin) else: assert not out @@ -124,7 +124,7 @@ def test_capturing_basic_api(self, method): print("hello") capman.suspend() out, err = capman.read() - if method != _CaptureMethod.NO: + if method != CaptureMethod.NO: assert out == "hello\n" capman.stop_capturing() finally: @@ -133,7 +133,7 @@ def test_capturing_basic_api(self, method): def test_init_capturing(self): capouter = StdCaptureFD() try: - capman = CaptureManager(_CaptureMethod.FD) + capman = CaptureManager(CaptureMethod.FD) capman.start_capturing() pytest.raises(AssertionError, capman.start_capturing) capman.stop_capturing() @@ -750,7 +750,7 @@ def test_fdcapture_invalid_fd_without_fd_reuse(self, tmp_path): @pytest.mark.unit() def test__get_multicapture() -> None: - assert isinstance(_get_multicapture(_CaptureMethod.NO), MultiCapture) + assert isinstance(_get_multicapture(CaptureMethod.NO), MultiCapture) pytest.raises(ValueError, _get_multicapture, "unknown").match( r"^unknown capturing method: 'unknown'" ) diff --git a/tests/test_collect.py b/tests/test_collect.py index c83939283..a462bf7d6 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -31,7 +31,7 @@ def task_write_text(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Relative paths work.") - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.collection_reports[0].outcome == CollectionOutcome.SUCCESS assert tmp_path.joinpath("out.txt").read_text() == "Relative paths work." @@ -49,7 +49,7 @@ def task_with_non_path_dependency(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.COLLECTION_FAILED assert session.collection_reports[0].outcome == CollectionOutcome.FAIL @@ -70,7 +70,7 @@ def task_with_non_path_dependency(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.COLLECTION_FAILED assert session.collection_reports[0].outcome == CollectionOutcome.FAIL @@ -114,7 +114,7 @@ def task_1(depends_on, produces): def test_collect_same_task_different_ways(tmp_path, path_extension): tmp_path.joinpath("task_module.py").write_text("def task_passes(): pass") - session = build({"paths": tmp_path.joinpath(path_extension)}) + session = build(paths=tmp_path.joinpath(path_extension)) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 @@ -141,7 +141,7 @@ def test_collect_files_w_custom_file_name_pattern( for file in task_files: tmp_path.joinpath(file).write_text("def task_example(): pass") - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.tasks) == expected_collected_tasks @@ -264,7 +264,7 @@ def task_example(path_in = Path("in.txt"), produces = Path("out.txt")): tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("hello") - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 @@ -278,7 +278,7 @@ def test_collect_tasks_from_modules_with_the_same_name(tmp_path): tmp_path.joinpath("b").mkdir() tmp_path.joinpath("a", "task_module.py").write_text("def task_a(): pass") tmp_path.joinpath("b", "task_module.py").write_text("def task_a(): pass") - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert len(session.collection_reports) == 2 assert all( report.outcome == CollectionOutcome.SUCCESS @@ -306,7 +306,7 @@ def task_my_task(): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) outcome = session.collection_reports[0].outcome assert outcome == CollectionOutcome.SUCCESS @@ -368,7 +368,7 @@ def task_example( ... """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.COLLECTION_FAILED assert len(session.tasks) == 0 @@ -395,7 +395,7 @@ def task_example( tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("input_1.txt").touch() tmp_path.joinpath("input_2.txt").touch() - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.COLLECTION_FAILED assert len(session.tasks) == 0 @@ -439,7 +439,7 @@ def task_example( """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": [tmp_path]}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK product = session.tasks[0].produces["path"] assert product.name == "product" diff --git a/tests/test_config.py b/tests/test_config.py index 320f078ce..ed45baec5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -9,7 +9,7 @@ @pytest.mark.end_to_end() def test_debug_pytask(capsys, tmp_path): - session = build({"paths": tmp_path, "debug_pytask": True}) + session = build(paths=tmp_path, debug_pytask=True) assert session.exit_code == ExitCode.OK @@ -31,7 +31,7 @@ def test_pass_config_to_cli(tmp_path): """ tmp_path.joinpath("pyproject.toml").write_text(textwrap.dedent(config)) - session = build({"config": tmp_path.joinpath("pyproject.toml"), "paths": tmp_path}) + session = build(config=tmp_path.joinpath("pyproject.toml"), paths=tmp_path) assert session.exit_code == ExitCode.OK assert "elton" in session.config["markers"] @@ -55,7 +55,7 @@ def test_passing_paths_via_configuration_file(tmp_path, file_or_folder): "def task_passes(): pass" ) - session = build({"config": tmp_path.joinpath("pyproject.toml")}) + session = build(config=tmp_path.joinpath("pyproject.toml")) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 diff --git a/tests/test_execute.py b/tests/test_execute.py index 8ce120e25..18d904a4f 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -9,7 +9,7 @@ from pathlib import Path import pytest -from _pytask.capture import _CaptureMethod +from _pytask.capture import CaptureMethod from _pytask.exceptions import NodeNotFoundError from pytask import build from pytask import cli @@ -46,7 +46,7 @@ def task_example(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.FAILED assert len(session.execution_reports) == 1 @@ -103,7 +103,7 @@ def task_3(depends_on): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.FAILED assert sum(i.outcome == TaskOutcome.SUCCESS for i in session.execution_reports) == 2 @@ -138,7 +138,7 @@ def task_example(depends_on, produces): for dependency in dependencies: tmp_path.joinpath(dependency).touch() - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK @@ -157,7 +157,7 @@ def task_example(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Here I am. Once again.") - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert tmp_path.joinpath("out.txt").read_text() == "Here I am. Once again." @@ -243,7 +243,7 @@ def task_example(depends_on, produces): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK @@ -257,7 +257,7 @@ def task_3(): raise Exception """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path, "max_failures": n_failures}) + session = build(paths=tmp_path, max_failures=n_failures) assert len(session.tasks) == 3 assert len(session.execution_reports) == n_failures @@ -273,9 +273,7 @@ def task_3(): raise Exception """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build( - {"paths": tmp_path, "stop_after_first_failure": stop_after_first_failure} - ) + session = build(paths=tmp_path, stop_after_first_failure=stop_after_first_failure) assert len(session.tasks) == 3 assert len(session.execution_reports) == 1 if stop_after_first_failure else 3 @@ -296,7 +294,7 @@ def task_y(): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert session.execution_reports[0].task.name.endswith("task_z") @@ -390,9 +388,9 @@ def task_example(): @pytest.mark.end_to_end() def test_task_executed_with_force_although_unchanged(tmp_path): tmp_path.joinpath("task_module.py").write_text("def task_example(): pass") - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS - session = build({"paths": tmp_path, "force": True}) + session = build(paths=tmp_path, force=True) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS @@ -436,7 +434,7 @@ def task_example(path_to_file: Annotated[Path, Product] = Path("out.txt")) -> No """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path, "capture": _CaptureMethod.NO}) + session = build(paths=tmp_path, capture=CaptureMethod.NO) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 @@ -459,7 +457,7 @@ def task_example( """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path, "capture": _CaptureMethod.NO}) + session = build(paths=tmp_path, capture=CaptureMethod.NO) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 diff --git a/tests/test_ignore.py b/tests/test_ignore.py index bf76f54bf..48c5af53a 100644 --- a/tests/test_ignore.py +++ b/tests/test_ignore.py @@ -16,7 +16,7 @@ def test_ignore_default_paths(tmp_path, ignored_folder): tmp_path.joinpath(folder).mkdir() tmp_path.joinpath(folder, "task_module.py").write_text("def task_d(): pass") - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 0 @@ -32,7 +32,7 @@ def test_ignore_paths(tmp_path, ignore, new_line): ) tmp_path.joinpath("pyproject.toml").write_text(config) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 0 if ignore else len(session.tasks) == 1 diff --git a/tests/test_mark.py b/tests/test_mark.py index 82f05a1d0..f9ccf93a8 100644 --- a/tests/test_mark.py +++ b/tests/test_mark.py @@ -109,7 +109,7 @@ def task_two(): """ ) ) - session = build({"paths": tmp_path, "marker_expression": expr}) + session = build(paths=tmp_path, marker_expression=expr) tasks_that_run = [ report.task.name.rsplit("::")[1] @@ -152,7 +152,7 @@ def task_no_2(): """ ) ) - session = build({"paths": tmp_path, "expression": expr}) + session = build(paths=tmp_path, expression=expr) assert session.exit_code == ExitCode.OK tasks_that_run = [ @@ -184,7 +184,7 @@ def task_func(arg=arg): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path, "expression": expr}) + session = build(paths=tmp_path, expression=expr) assert session.exit_code == ExitCode.OK tasks_that_run = [ @@ -233,7 +233,7 @@ def test_keyword_option_wrong_arguments( tmp_path.joinpath("task_module.py").write_text( textwrap.dedent("def task_func(arg): pass") ) - session = build({"paths": tmp_path, option: expr}) + session = build(paths=tmp_path, **{option: expr}) assert session.exit_code == ExitCode.DAG_FAILED captured = capsys.readouterr() diff --git a/tests/test_mark_cli.py b/tests/test_mark_cli.py index 2a6f5ce77..23c0f54b5 100644 --- a/tests/test_mark_cli.py +++ b/tests/test_mark_cli.py @@ -48,5 +48,5 @@ def test_marker_names(tmp_path, marker_name): markers = ['{marker_name}'] """ tmp_path.joinpath("pyproject.toml").write_text(textwrap.dedent(toml)) - session = build({"paths": tmp_path, "markers": True}) + session = build(paths=tmp_path, markers=True) assert session.exit_code == ExitCode.CONFIGURATION_FAILED diff --git a/tests/test_persist.py b/tests/test_persist.py index 668e11901..9dd3f4f13 100644 --- a/tests/test_persist.py +++ b/tests/test_persist.py @@ -20,7 +20,7 @@ class DummyClass: @pytest.mark.end_to_end() def test_persist_marker_is_set(tmp_path): - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert "persist" in session.config["markers"] @@ -46,7 +46,7 @@ def task_dummy(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("I'm not the reason you care.") - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 @@ -56,7 +56,7 @@ def task_dummy(depends_on, produces): tmp_path.joinpath("out.txt").write_text("Never again in despair.") - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 @@ -72,7 +72,7 @@ def task_dummy(depends_on, produces): modification_time = session.get(State, (task_id, node_id)).modification_time assert float(modification_time) == tmp_path.joinpath("out.txt").stat().st_mtime - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 @@ -97,7 +97,7 @@ def task_dummy(depends_on, produces): "They say oh my god I see the way you shine." ) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.execution_reports) == 1 diff --git a/tests/test_profile.py b/tests/test_profile.py index b3ba99a21..a516e0e88 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -22,7 +22,7 @@ def task_example(): time.sleep(2) """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert len(session.tasks) == 1 diff --git a/tests/test_shared.py b/tests/test_shared.py index b994d20b8..2abbcfe9b 100644 --- a/tests/test_shared.py +++ b/tests/test_shared.py @@ -27,7 +27,7 @@ def test_parse_markers(tmp_path): """ tmp_path.joinpath("pyproject.toml").write_text(textwrap.dedent(toml)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK assert "a1" in session.config["markers"] diff --git a/tests/test_skipping.py b/tests/test_skipping.py index a825f0157..653847794 100644 --- a/tests/test_skipping.py +++ b/tests/test_skipping.py @@ -30,10 +30,10 @@ def task_dummy(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert isinstance(session.execution_reports[0].exc_info[1], SkippedUnchanged) @@ -50,12 +50,12 @@ def task_dummy(depends_on, produces): tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) tmp_path.joinpath("in.txt").write_text("Original content of in.txt.") - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SUCCESS assert tmp_path.joinpath("out.txt").read_text() == "Original content of in.txt." - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SKIP_UNCHANGED assert isinstance(session.execution_reports[0].exc_info[1], SkippedUnchanged) @@ -77,7 +77,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.FAIL assert isinstance(session.execution_reports[0].exc_info[1], Exception) @@ -101,7 +101,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SKIP assert isinstance(session.execution_reports[0].exc_info[1], Skipped) @@ -124,7 +124,7 @@ def task_first(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.execution_reports[0].outcome == TaskOutcome.SKIP assert isinstance(session.execution_reports[0].exc_info[1], Skipped) @@ -173,7 +173,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) node = session.collection_reports[0].node assert len(node.markers) == 1 assert node.markers[0].name == "skipif" @@ -203,7 +203,7 @@ def task_second(depends_on): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) node = session.collection_reports[0].node assert len(node.markers) == 1 @@ -234,7 +234,7 @@ def task_second(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) node = session.collection_reports[0].node assert len(node.markers) == 2 assert node.markers[0].name == "skipif" diff --git a/tests/test_task.py b/tests/test_task.py index fe978edff..0c71a5f63 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -23,7 +23,7 @@ def {func_name}(produces): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.OK @@ -434,7 +434,7 @@ def task_func(i=i): pass """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == ExitCode.COLLECTION_FAILED assert isinstance(session.collection_reports[0].exc_info[1], ValueError) diff --git a/tests/test_tree_util.py b/tests/test_tree_util.py index 355844b04..107552cc8 100644 --- a/tests/test_tree_util.py +++ b/tests/test_tree_util.py @@ -40,7 +40,7 @@ def task_example(): """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path}) + session = build(paths=tmp_path) assert session.exit_code == exit_code diff --git a/tests/test_warnings.py b/tests/test_warnings.py index 4d973d31b..a5a43f2b7 100644 --- a/tests/test_warnings.py +++ b/tests/test_warnings.py @@ -46,7 +46,7 @@ def task_example(): """ tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) - session = build({"paths": tmp_path, "disable_warnings": disable_warnings}) + session = build(paths=tmp_path, disable_warnings=disable_warnings) assert session.exit_code == ExitCode.OK if disable_warnings: From 7613da532d0f57bab134d2d077c91a8140bd11f7 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Tue, 29 Aug 2023 23:18:00 +0200 Subject: [PATCH 03/25] to changes. --- docs/source/changes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/changes.md b/docs/source/changes.md index eaa1292d0..ec4e7f252 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -28,6 +28,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`408` removes `.value` from `Node` protocol. - {pull}`409` make `.from_annot` an optional feature of nodes. - {pull}`410` allows to pass functions to `PythonNode(hash=...)`. +- {pull}`411` implements a new functional interface. ## 0.3.2 - 2023-06-07 From c1c5f7715a78027a1f920ff287c2bc7292fda45c Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 1 Sep 2023 16:15:31 +0200 Subject: [PATCH 04/25] Continue with functional interface. --- src/_pytask/build.py | 5 ++++ src/_pytask/collect.py | 48 +++++++++++++++++++++++++++-- src/_pytask/collect_utils.py | 11 ++++++- src/_pytask/nodes.py | 2 +- src/_pytask/session.py | 2 +- tests/test_debugging.py | 2 +- tests/test_execute.py | 58 ++++++++++++++++++++++++++++++++++++ 7 files changed, 122 insertions(+), 6 deletions(-) diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 392218e58..95ece1342 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -4,6 +4,7 @@ import sys from pathlib import Path from typing import Any +from typing import Callable from typing import Iterable from typing import TYPE_CHECKING @@ -18,6 +19,7 @@ from _pytask.exceptions import ConfigurationError from _pytask.exceptions import ExecutionError from _pytask.exceptions import ResolvingDependenciesError +from _pytask.nodes import Task from _pytask.outcomes import ExitCode from _pytask.pluginmanager import get_plugin_manager from _pytask.session import Session @@ -38,6 +40,7 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: def build( # noqa: C901, PLR0912, PLR0913, PLR0915 + *, capture: CaptureMethod = CaptureMethod.NO, check_casing_of_paths: bool = True, config: Path | None = None, @@ -63,6 +66,7 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 sort_table: bool = True, stop_after_first_failure: bool = False, strict_markers: bool = False, + tasks: Callable[..., Any] | Task | Iterable[Callable[..., Any] | Task] = (), task_files: str | Iterable[str] = "task_*.py", trace: bool = False, verbose: int = 1, @@ -118,6 +122,7 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 "sort_table": sort_table, "stop_after_first_failure": stop_after_first_failure, "strict_markers": strict_markers, + "tasks": tasks, "task_files": task_files, "trace": trace, "verbose": verbose, diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 26db423fd..d450ee2d3 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -15,6 +15,7 @@ from _pytask.collect_utils import parse_products_from_task_function from _pytask.config import hookimpl from _pytask.config import IS_FILE_SYSTEM_CASE_SENSITIVE +from _pytask.console import _get_file from _pytask.console import console from _pytask.console import create_summary_panel from _pytask.console import format_task_id @@ -33,6 +34,7 @@ from _pytask.session import Session from _pytask.shared import find_duplicates from _pytask.shared import reduce_node_name +from _pytask.task_utils import task as task_decorator from _pytask.traceback import render_exc_info from rich.text import Text @@ -43,6 +45,13 @@ def pytask_collect(session: Session) -> bool: session.collection_start = time.time() _collect_from_paths(session) + _collect_from_tasks(session) + + session.tasks.extend( + i.node + for i in session.collection_reports + if i.outcome == CollectionOutcome.SUCCESS and isinstance(i.node, Task) + ) try: session.hook.pytask_collect_modify_tasks(session=session, tasks=session.tasks) @@ -72,10 +81,43 @@ def _collect_from_paths(session: Session) -> None: if reports: session.collection_reports.extend(reports) - session.tasks.extend( - i.node for i in reports if i.outcome == CollectionOutcome.SUCCESS + + +def _collect_from_tasks(session: Session) -> None: + """Collect tasks from user provided tasks via the functional interface.""" + for raw_task in session.config.get("tasks", ()): + if isinstance(raw_task, Task): + report = session.hook.pytask_collect_task_protocol( + session=session, + reports=session.collection_reports, + path=None, + name=None, + obj=raw_task, ) + if not callable(raw_task): + raise ValueError("Not a function.") + + if not hasattr(raw_task, "pytask_meta"): + raw_task = task_decorator()(raw_task) # noqa: PLW2901 + + path = _get_file(raw_task) + if path.name == "": + path = None + + name = raw_task.pytask_meta.name + + report = session.hook.pytask_collect_task_protocol( + session=session, + reports=session.collection_reports, + path=path, + name=name, + obj=raw_task, + ) + + if report is not None: + session.collection_reports.append(report) + @hookimpl def pytask_ignore_collect(path: Path, config: dict[str, Any]) -> bool: @@ -187,6 +229,8 @@ def pytask_collect_task( produces=products, markers=markers, ) + if isinstance(obj, Task): + return obj return None diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 5d0bf273c..dea4dd700 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -262,8 +262,17 @@ def parse_dependencies_from_task_function( parameters_with_product_annot = _find_args_with_product_annotation(obj) parameters_with_node_annot = _find_args_with_node_annotation(obj) + # Complete kwargs with node annotations, when no value is given by kwargs. + for name in list(parameters_with_node_annot): + if name not in kwargs: + kwargs[name] = parameters_with_node_annot[name] + parameters_with_node_annot.pop(name) + for parameter_name, value in kwargs.items(): - if parameter_name in parameters_with_product_annot: + if ( + parameter_name in parameters_with_product_annot + or parameter_name == "return" + ): continue if parameter_name == "depends_on": diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index c653304aa..a5953f2fc 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -40,7 +40,7 @@ class Task(MetaNode): base_name: str """The base name of the task.""" - path: Path + path: Path | None """Path to the file where the task was defined.""" function: Callable[..., Any] """The task function.""" diff --git a/src/_pytask/session.py b/src/_pytask/session.py index f486a19a2..bab5e499d 100644 --- a/src/_pytask/session.py +++ b/src/_pytask/session.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: from _pytask.report import CollectionReport from _pytask.report import ExecutionReport - from _ptytask.report import DagReport + from _pytask.report import DagReport from _pytask.nodes import Task diff --git a/tests/test_debugging.py b/tests/test_debugging.py index 5502bac49..aece1cd78 100644 --- a/tests/test_debugging.py +++ b/tests/test_debugging.py @@ -473,7 +473,7 @@ def test_set_trace_is_returned_after_pytask_finishes(tmp_path): import pytask def test_function(): - pytask.build({{"paths": "{tmp_path.as_posix()}"}}) + pytask.build(paths={tmp_path.as_posix()!r}) breakpoint() """ tmp_path.joinpath("test_example.py").write_text(textwrap.dedent(source)) diff --git a/tests/test_execute.py b/tests/test_execute.py index 18d904a4f..9bc329694 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -704,3 +704,61 @@ def task_example() -> Annotated[Dict[str, str], nodes]: tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == ExitCode.OK + + +def test_execute_tasks_and_pass_values_only_by_python_nodes(runner, tmp_path): + source = """ + from _pytask.nodes import PathNode + from pytask import PythonNode + from typing_extensions import Annotated + from pathlib import Path + + + node_text = PythonNode(name="text") + + + def task_create_text() -> Annotated[int, node_text]: + return "This is the text." + + node_file = PathNode.from_path(Path(__file__).parent.joinpath("file.txt")) + + def task_create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]: + return text + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("file.txt").read_text() == "This is the text." + + +def test_execute_tasks_via_functional_api(tmp_path): + source = """ + from _pytask.nodes import PathNode + import pytask + from pytask import PythonNode + from typing_extensions import Annotated + from pathlib import Path + + + node_text = PythonNode(name="text", hash=True) + + def create_text() -> Annotated[int, node_text]: + return "This is the text." + + node_file = PathNode.from_path(Path(__file__).parent.joinpath("file.txt")) + + def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]: + return text + + if __name__ == "__main__": + session = pytask.build(tasks=[create_file, create_text]) + + assert len(session.tasks) == 2 + assert len(session.dag.nodes) == 4 + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + result = subprocess.run( + ("python", tmp_path.joinpath("task_module.py").as_posix()), check=False + ) + assert result.returncode == ExitCode.OK + assert tmp_path.joinpath("file.txt").read_text() == "This is the text." From 00d78cdba6134079ecf193766721937426a2bed6 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 1 Sep 2023 18:05:26 +0200 Subject: [PATCH 05/25] Differentiate between unset value and None with sentinel. --- src/_pytask/collect_command.py | 11 +++++++-- src/_pytask/collect_utils.py | 2 +- src/_pytask/execute.py | 2 ++ src/_pytask/node_protocols.py | 4 +++- src/_pytask/nodes.py | 31 +++++++++++++++---------- src/_pytask/profile.py | 2 ++ src/_pytask/shared.py | 2 ++ src/_pytask/typing.py | 42 ++++++++++++++++++++++++++++++++++ src/pytask/__init__.py | 3 ++- tests/test_collect_utils.py | 2 +- 10 files changed, 83 insertions(+), 18 deletions(-) create mode 100644 src/_pytask/typing.py diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 4a66b8e72..cf93a71ef 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -27,6 +27,7 @@ from _pytask.pluginmanager import get_plugin_manager from _pytask.session import Session from _pytask.tree_util import tree_leaves +from _pytask.typing import no_value from rich.text import Text from rich.tree import Tree @@ -125,10 +126,14 @@ def _find_common_ancestor_of_all_nodes( all_paths.append(task.path) if show_nodes: all_paths.extend( - x.path for x in tree_leaves(task.depends_on) if isinstance(x, PPathNode) + x.path + for x in tree_leaves(task.depends_on) + if isinstance(x, PPathNode) and x.path is not no_value ) all_paths.extend( - x.path for x in tree_leaves(task.produces) if isinstance(x, PPathNode) + x.path + for x in tree_leaves(task.produces) + if isinstance(x, PPathNode) and x.path is not no_value ) common_ancestor = find_common_ancestor(*all_paths, *paths) @@ -206,6 +211,7 @@ def _print_collected_tasks( sorted_nodes = sorted(nodes, key=lambda x: x.name) for node in sorted_nodes: if isinstance(node, PPathNode): + assert node.path is not no_value if node.path.as_posix() in node.name: reduced_node_name = str( relative_to(node.path, common_ancestor) @@ -225,6 +231,7 @@ def _print_collected_tasks( tree_leaves(task.produces), key=lambda x: getattr(x, "path", x.name) ): if isinstance(node, PPathNode): + assert node.path is not no_value reduced_node_name = str(relative_to(node.path, common_ancestor)) url_style = create_url_style_for_path( node.path, editor_url_scheme diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index dea4dd700..3d7f6b765 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -18,7 +18,6 @@ from _pytask.mark_utils import remove_marks from _pytask.models import NodeInfo from _pytask.node_protocols import Node -from _pytask.nodes import ProductType from _pytask.nodes import PythonNode from _pytask.shared import find_duplicates from _pytask.task_utils import parse_keyword_arguments_from_signature_defaults @@ -26,6 +25,7 @@ from _pytask.tree_util import tree_leaves from _pytask.tree_util import tree_map from _pytask.tree_util import tree_map_with_path +from _pytask.typing import ProductType from attrs import define from attrs import field from typing_extensions import Annotated diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index 6603d33e2..1e5868d7b 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -34,6 +34,7 @@ from _pytask.traceback import remove_traceback_from_exc_info from _pytask.traceback import render_exc_info from _pytask.tree_util import tree_map +from _pytask.typing import no_value from rich.text import Text @@ -130,6 +131,7 @@ def pytask_execute_task_setup(session: Session, task: Task) -> None: for product in session.dag.successors(task.name): node = session.dag.nodes[product]["node"] if isinstance(node, PPathNode): + assert node.path is not no_value node.path.parent.mkdir(parents=True, exist_ok=True) would_be_executed = has_mark(task, "would_be_executed") diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py index b8660a874..6e7c825c9 100644 --- a/src/_pytask/node_protocols.py +++ b/src/_pytask/node_protocols.py @@ -6,6 +6,8 @@ from typing import Protocol from typing import runtime_checkable +from _pytask.typing import NoValue + @runtime_checkable class MetaNode(Protocol): @@ -46,4 +48,4 @@ class PPathNode(Node, Protocol): """ - path: Path + path: Path | NoValue diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index a5953f2fc..a39740a85 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -14,6 +14,8 @@ from _pytask.tree_util import PyTree from _pytask.tree_util import tree_leaves from _pytask.tree_util import tree_structure +from _pytask.typing import no_value +from _pytask.typing import NoValue from attrs import define from attrs import field @@ -22,16 +24,7 @@ from _pytask.mark import Mark -__all__ = ["PathNode", "Product", "Task"] - - -@define(frozen=True) -class ProductType: - """A class to mark products.""" - - -Product = ProductType() -"""ProductType: A singleton to mark products in annotations.""" +__all__ = ["PathNode", "PythonNode", "Task"] @define(kw_only=True) @@ -107,7 +100,7 @@ class PathNode(PPathNode): name: str = "" """Name of the node which makes it identifiable in the DAG.""" - path: Path | None = None + path: Path | NoValue = no_value """The path to the file.""" def from_annot(self, value: Path) -> None: @@ -122,6 +115,8 @@ def task_example(value: Annotated[Any, PathNode(name="value")]): """ + if self.path is not no_value: + raise ValueError("'path' is already defined.") if not isinstance(value, Path): raise TypeError("'value' must be a 'pathlib.Path'.") if not self.name: @@ -146,16 +141,22 @@ def state(self) -> str | None: The state is given by the modification timestamp. """ + if self.path is no_value: + return None if self.path.exists(): return str(self.path.stat().st_mtime) return None def load(self) -> Path: """Load the value.""" + if self.path is no_value: + raise ValueError("PathNode has no path defined.") return self.path def save(self, value: bytes | str) -> None: """Save strings or bytes to file.""" + if self.path is no_value: + raise TypeError("'path' must be a 'pathlib.Path' to store data.") if isinstance(value, str): self.path.write_text(value) elif isinstance(value, bytes): @@ -172,13 +173,15 @@ class PythonNode(Node): name: str = "" """Name of the node.""" - value: Any = None + value: Any | NoValue = no_value """Value of the node.""" hash: bool | Callable[[Any], bool] = False # noqa: A003 """Whether the value should be hashed to determine the state.""" def load(self) -> Any: """Load the value.""" + if self.value is no_value: + raise ValueError("PythonNode has no value defined.") return self.value def save(self, value: Any) -> None: @@ -198,6 +201,8 @@ def task_example(value: Annotated[Any, PythonNode(hash=True)]): ... """ + if self.value is not no_value: + raise ValueError("'value' is already defined.") self.value = value def state(self) -> str | None: @@ -215,6 +220,8 @@ def state(self) -> str | None: random integer and it would confuse users. """ + if self.value is no_value: + return None if self.hash: if callable(self.hash): return str(self.hash(self.value)) diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 0ccbfbdf0..b35673c01 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -31,6 +31,7 @@ from _pytask.report import ExecutionReport from _pytask.session import Session from _pytask.traceback import render_exc_info +from _pytask.typing import no_value from rich.table import Table from sqlalchemy import Column from sqlalchemy import Float @@ -230,6 +231,7 @@ def pytask_profile_add_info_on_task( node = session.dag.nodes[successor]["node"] if isinstance(node, PPathNode): with suppress(FileNotFoundError): + assert node.path is not no_value sum_bytes += node.path.stat().st_size profile[task.name]["Size of Products"] = _to_human_readable_size( diff --git a/src/_pytask/shared.py b/src/_pytask/shared.py index 465f15b35..4949991f8 100644 --- a/src/_pytask/shared.py +++ b/src/_pytask/shared.py @@ -16,6 +16,7 @@ from _pytask.path import find_closest_ancestor from _pytask.path import find_common_ancestor from _pytask.path import relative_to +from _pytask.typing import no_value def to_list(scalar_or_iter: Any) -> list[Any]: @@ -70,6 +71,7 @@ def reduce_node_name(node: MetaNode, paths: Sequence[str | Path]) -> str: """ if isinstance(node, (PPathNode, Task)): + assert node.path is not no_value ancestor = find_closest_ancestor(node.path, paths) if ancestor is None: try: diff --git a/src/_pytask/typing.py b/src/_pytask/typing.py new file mode 100644 index 000000000..bcdb8dd01 --- /dev/null +++ b/src/_pytask/typing.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from enum import Enum +from typing import Literal + +from attr import define + + +__all__ = ["no_value", "NoValue"] + + +class _NoValue(Enum): + """Sentinel value when no value was provided. + + For nodes, we need a different default value than :obj:`None` to signal that the + user provided no value. + + We make this an Enum + + - because it round-trips through pickle correctly (see GH#pandas/40397) + - because mypy does not understand singletons + + """ + + no_value = "NO_VALUE" + + def __repr__(self) -> str: + return "" + + +# Note: no_value is exported to the public API in pytask +no_value = _NoValue.no_value +NoValue = Literal[_NoValue.no_value] + + +@define(frozen=True) +class ProductType: + """A class to mark products.""" + + +Product = ProductType() +"""ProductType: A singleton to mark products in annotations.""" diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py index f71c15d98..2822677ed 100644 --- a/src/pytask/__init__.py +++ b/src/pytask/__init__.py @@ -38,7 +38,6 @@ from _pytask.models import NodeInfo from _pytask.node_protocols import MetaNode from _pytask.nodes import PathNode -from _pytask.nodes import Product from _pytask.nodes import PythonNode from _pytask.nodes import Task from _pytask.outcomes import CollectionOutcome @@ -59,6 +58,7 @@ from _pytask.traceback import remove_internal_traceback_frames_from_exc_info from _pytask.traceback import remove_traceback_from_exc_info from _pytask.traceback import render_exc_info +from _pytask.typing import Product from _pytask.warnings_utils import parse_warning_filter from _pytask.warnings_utils import warning_record_to_str from _pytask.warnings_utils import WarningReport @@ -92,6 +92,7 @@ "NodeInfo", "NodeNotCollectedError", "NodeNotFoundError", + "PathNode", "Persisted", "Product", "PytaskError", diff --git a/tests/test_collect_utils.py b/tests/test_collect_utils.py index e4dbc4c10..b58d21118 100644 --- a/tests/test_collect_utils.py +++ b/tests/test_collect_utils.py @@ -12,9 +12,9 @@ from _pytask.collect_utils import _find_args_with_product_annotation from _pytask.collect_utils import _merge_dictionaries from _pytask.collect_utils import _Placeholder +from _pytask.typing import Product from pytask import depends_on from pytask import produces -from pytask import Product from typing_extensions import Annotated From c614100dcc071a04fadc30bac2798f8fe0cbbded Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 1 Sep 2023 18:07:21 +0200 Subject: [PATCH 06/25] Maybe fix test. --- tests/test_execute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_execute.py b/tests/test_execute.py index 9bc329694..ca2a752d3 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -758,7 +758,8 @@ def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]: """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) result = subprocess.run( - ("python", tmp_path.joinpath("task_module.py").as_posix()), check=False + ("python", tmp_path.joinpath("task_module.py").as_posix(), "-v", "0"), + check=False, ) assert result.returncode == ExitCode.OK assert tmp_path.joinpath("file.txt").read_text() == "This is the text." From fa609cf5434f4ec3df7028e6c062e114fa11c607 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 1 Sep 2023 18:13:15 +0200 Subject: [PATCH 07/25] Maybe fix test. --- tests/test_execute.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_execute.py b/tests/test_execute.py index ca2a752d3..89126db0b 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -751,15 +751,14 @@ def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]: return text if __name__ == "__main__": - session = pytask.build(tasks=[create_file, create_text]) + session = pytask.build(tasks=[create_file, create_text], verbose=0) assert len(session.tasks) == 2 assert len(session.dag.nodes) == 4 """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) result = subprocess.run( - ("python", tmp_path.joinpath("task_module.py").as_posix(), "-v", "0"), - check=False, + ("python", tmp_path.joinpath("task_module.py").as_posix()), check=False ) assert result.returncode == ExitCode.OK assert tmp_path.joinpath("file.txt").read_text() == "This is the text." From 73c1a51584577b613818434cd3abb228ce5ab22d Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 1 Sep 2023 18:37:10 +0200 Subject: [PATCH 08/25] Maybe fix test. --- tests/test_execute.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_execute.py b/tests/test_execute.py index 89126db0b..b16fb1a5f 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -751,14 +751,16 @@ def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]: return text if __name__ == "__main__": - session = pytask.build(tasks=[create_file, create_text], verbose=0) + session = pytask.build(tasks=[create_file, create_text]) assert len(session.tasks) == 2 assert len(session.dag.nodes) == 4 """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) result = subprocess.run( - ("python", tmp_path.joinpath("task_module.py").as_posix()), check=False + ("python", tmp_path.joinpath("task_module.py").as_posix()), + check=False, + capture_output=True, ) assert result.returncode == ExitCode.OK assert tmp_path.joinpath("file.txt").read_text() == "This is the text." From fcaaeae780e869f253e812f14193d174b33fe56e Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 1 Sep 2023 18:55:20 +0200 Subject: [PATCH 09/25] Maybe fix test. --- tests/test_execute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_execute.py b/tests/test_execute.py index b16fb1a5f..867172cb4 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -760,7 +760,7 @@ def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]: result = subprocess.run( ("python", tmp_path.joinpath("task_module.py").as_posix()), check=False, - capture_output=True, + env={**os.environ, "TERM": "unknown"}, ) assert result.returncode == ExitCode.OK assert tmp_path.joinpath("file.txt").read_text() == "This is the text." From e9778079d1697c318426baa6e2571f94d1bb9b27 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 1 Sep 2023 19:01:51 +0200 Subject: [PATCH 10/25] Maybe fix test. --- tests/test_execute.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_execute.py b/tests/test_execute.py index 867172cb4..c9f5b8d8b 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -731,6 +731,8 @@ def task_create_file(text: Annotated[int, node_text]) -> Annotated[str, node_fil assert tmp_path.joinpath("file.txt").read_text() == "This is the text." +@pytest.mark.end_to_end() +@pytest.mark.xfail(sys.platform == "win32", reason="Decoding issues in Gitlab Actions.") def test_execute_tasks_via_functional_api(tmp_path): source = """ from _pytask.nodes import PathNode @@ -758,9 +760,7 @@ def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]: """ tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) result = subprocess.run( - ("python", tmp_path.joinpath("task_module.py").as_posix()), - check=False, - env={**os.environ, "TERM": "unknown"}, + ("python", tmp_path.joinpath("task_module.py").as_posix()), check=False ) assert result.returncode == ExitCode.OK assert tmp_path.joinpath("file.txt").read_text() == "This is the text." From 8b9e946f5c15465b93d4cf540f1e5ae20f2ca807 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 1 Sep 2023 22:02:34 +0200 Subject: [PATCH 11/25] TMEP COMMIT. --- src/_pytask/build.py | 5 ----- src/_pytask/collect.py | 6 ++++++ src/_pytask/console.py | 20 ++++++++++++++++++++ src/_pytask/nodes.py | 5 ++++- src/_pytask/report.py | 1 - 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 95ece1342..dc3874ea2 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -78,11 +78,6 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 command line interface. It can also be used to run pytask interactively. Pass configuration in a dictionary. - Parameters - ---------- - capture - Defines the capture method. - Returns ------- session : _pytask.session.Session diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index d450ee2d3..5b133f593 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -19,6 +19,7 @@ from _pytask.console import console from _pytask.console import create_summary_panel from _pytask.console import format_task_id +from _pytask.console import is_jupyter from _pytask.exceptions import CollectionError from _pytask.mark_utils import has_mark from _pytask.models import NodeInfo @@ -105,6 +106,9 @@ def _collect_from_tasks(session: Session) -> None: if path.name == "": path = None + if is_jupyter(): + path = None + name = raw_task.pytask_meta.name report = session.hook.pytask_collect_task_protocol( @@ -364,6 +368,8 @@ def pytask_collect_log( console.print(f"Collected {len(tasks)} task{'' if len(tasks) == 1 else 's'}.") + print(tasks) + failed_reports = [r for r in reports if r.outcome == CollectionOutcome.FAIL] if failed_reports: counts = count_outcomes(reports, CollectionOutcome) diff --git a/src/_pytask/console.py b/src/_pytask/console.py index 461cbb443..e90f98d40 100644 --- a/src/_pytask/console.py +++ b/src/_pytask/console.py @@ -297,3 +297,23 @@ def create_summary_panel( ) return panel + + +def is_jupyter() -> bool: # pragma: no cover + """Check if we're running in a Jupyter notebook.""" + try: + get_ipython # type: ignore[name-defined] + except NameError: + return False + ipython = get_ipython() # type: ignore[name-defined] + shell = ipython.__class__.__name__ + if ( + "google.colab" in str(ipython.__class__) + or os.getenv("DATABRICKS_RUNTIME_VERSION") + or shell == "ZMQInteractiveShell" + ): + return True # Jupyter notebook or qtconsole + elif shell == "TerminalInteractiveShell": + return False # Terminal running IPython + else: + return False # Other type (?) diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index a39740a85..61fa7cf31 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -55,7 +55,10 @@ class Task(MetaNode): def __attrs_post_init__(self: Task) -> None: """Change class after initialization.""" if self.name is None: - self.name = self.path.as_posix() + "::" + self.base_name + if self.path is None: + self.name = self.base_name + else: + self.name = self.path.as_posix() + "::" + self.base_name if self.short_name is None: self.short_name = self.name diff --git a/src/_pytask/report.py b/src/_pytask/report.py index 7df151fc3..185982912 100644 --- a/src/_pytask/report.py +++ b/src/_pytask/report.py @@ -37,7 +37,6 @@ def from_exception( exc_info: ExceptionInfo, node: MetaNode | None = None, ) -> CollectionReport: - exc_info = remove_internal_traceback_frames_from_exc_info(exc_info) return cls(outcome=outcome, node=node, exc_info=exc_info) From 0b18ad893eec728a8513a3fb9ba3a2f5b37c726b Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Mon, 4 Sep 2023 16:45:00 +0200 Subject: [PATCH 12/25] Make types easier. --- src/_pytask/path.py | 5 +---- tests/test_path.py | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/_pytask/path.py b/src/_pytask/path.py index 71c80b50d..3a55bf210 100644 --- a/src/_pytask/path.py +++ b/src/_pytask/path.py @@ -38,7 +38,7 @@ def relative_to( def find_closest_ancestor( - path: str | Path, potential_ancestors: Sequence[str | Path] + path: Path, potential_ancestors: Sequence[str | Path] ) -> Path: """Find the closest ancestor of a path. @@ -60,9 +60,6 @@ def find_closest_ancestor( 'folder/subfolder' """ - if isinstance(path, str): - path = Path(path) - closest_ancestor = None for ancestor in potential_ancestors: if isinstance(ancestor, str): diff --git a/tests/test_path.py b/tests/test_path.py index e55be2f79..fb9bf3611 100644 --- a/tests/test_path.py +++ b/tests/test_path.py @@ -37,7 +37,6 @@ def test_relative_to(path, source, include_source, expected): @pytest.mark.parametrize( ("path", "potential_ancestors", "expected"), [ - ("src/task.py", ["src", "bld"], Path("src")), (Path("src/task.py"), [Path("src"), Path("bld")], Path("src")), (Path("tasks/task.py"), [Path("src"), Path("bld")], None), (Path("src/ts/task.py"), [Path("src"), Path("src/ts")], Path("src/ts")), From 0373e6a3d5f7394bda90c544501ce37ad801cce9 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Mon, 4 Sep 2023 17:05:05 +0200 Subject: [PATCH 13/25] Simplify some path types in functions. --- src/_pytask/path.py | 23 ++++++++--------------- src/_pytask/shared.py | 4 ++-- tests/test_collect_command.py | 4 ++-- tests/test_path.py | 1 - 4 files changed, 12 insertions(+), 20 deletions(-) diff --git a/src/_pytask/path.py b/src/_pytask/path.py index 3a55bf210..6a913fb9d 100644 --- a/src/_pytask/path.py +++ b/src/_pytask/path.py @@ -10,9 +10,7 @@ from typing import Sequence -def relative_to( - path: str | Path, source: str | Path, include_source: bool = True -) -> Path: +def relative_to(path: Path, source: Path, include_source: bool = True) -> Path: """Make a path relative to another path. In contrast to :meth:`pathlib.Path.relative_to`, this function allows to keep the @@ -23,23 +21,21 @@ def relative_to( The default behavior of :mod:`pathlib` is to exclude the source path from the relative path. - >>> relative_to("folder/file.py", "folder", False).as_posix() + >>> relative_to(Path("folder/file.py"), Path("folder"), False).as_posix() 'file.py' To provide relative locations to users, it is sometimes more helpful to provide the source as an orientation. - >>> relative_to("folder/file.py", "folder").as_posix() + >>> relative_to(Path("folder/file.py"), Path("folder")).as_posix() 'folder/file.py' """ - source_name = Path(source).name if include_source else "" - return Path(source_name, Path(path).relative_to(source)) + source_name = source.name if include_source else "" + return Path(source_name, path.relative_to(source)) -def find_closest_ancestor( - path: Path, potential_ancestors: Sequence[str | Path] -) -> Path: +def find_closest_ancestor(path: Path, potential_ancestors: Sequence[Path]) -> Path: """Find the closest ancestor of a path. In case a path is the path to the task file itself, we return the path. @@ -62,9 +58,6 @@ def find_closest_ancestor( """ closest_ancestor = None for ancestor in potential_ancestors: - if isinstance(ancestor, str): - ancestor = Path(ancestor) # noqa: PLW2901 - if ancestor == path: closest_ancestor = path break @@ -87,11 +80,11 @@ def find_closest_ancestor( def find_common_ancestor_of_nodes(*names: str) -> Path: """Find the common ancestor from task names and nodes.""" - cleaned_names = [name.split("::")[0] for name in names] + cleaned_names = [Path(name.split("::")[0]) for name in names] return find_common_ancestor(*cleaned_names) -def find_common_ancestor(*paths: str | Path) -> Path: +def find_common_ancestor(*paths: Path) -> Path: """Find a common ancestor of many paths.""" return Path(os.path.commonpath(paths)) diff --git a/src/_pytask/shared.py b/src/_pytask/shared.py index f07a1b222..2685a5b62 100644 --- a/src/_pytask/shared.py +++ b/src/_pytask/shared.py @@ -63,7 +63,7 @@ def parse_paths(x: Any | None) -> list[Path] | None: return out -def reduce_node_name(node: MetaNode, paths: Sequence[str | Path]) -> str: +def reduce_node_name(node: MetaNode, paths: Sequence[Path]) -> str: """Reduce the node name. The whole name of the node - which includes the drive letter - can be very long @@ -86,7 +86,7 @@ def reduce_node_name(node: MetaNode, paths: Sequence[str | Path]) -> str: def reduce_names_of_multiple_nodes( - names: list[str], dag: nx.DiGraph, paths: Sequence[str | Path] + names: list[str], dag: nx.DiGraph, paths: Sequence[Path] ) -> list[str]: """Reduce the names of multiple nodes in the DAG.""" short_names = [] diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py index c0c16c46b..1bb9fb345 100644 --- a/tests/test_collect_command.py +++ b/tests/test_collect_command.py @@ -357,7 +357,7 @@ def function(depends_on, produces): # noqa: ARG001 @pytest.mark.unit() def test_print_collected_tasks_without_nodes(capsys): dictionary = { - "task_path.py": [ + Path("task_path.py"): [ Task( base_name="function", path=Path("task_path.py"), @@ -380,7 +380,7 @@ def test_print_collected_tasks_without_nodes(capsys): @pytest.mark.unit() def test_print_collected_tasks_with_nodes(capsys): dictionary = { - "task_path.py": [ + Path("task_path.py"): [ Task( base_name="function", path=Path("task_path.py"), diff --git a/tests/test_path.py b/tests/test_path.py index fb9bf3611..a1cbe3cf1 100644 --- a/tests/test_path.py +++ b/tests/test_path.py @@ -23,7 +23,6 @@ @pytest.mark.parametrize( ("path", "source", "include_source", "expected"), [ - ("src/hello.py", "src", True, Path("src/hello.py")), (Path("src/hello.py"), Path("src"), True, Path("src/hello.py")), (Path("src/hello.py"), Path("src"), False, Path("hello.py")), ], From 17394156b8232471954d186340adc2db543c281b Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 7 Sep 2023 09:22:44 +0200 Subject: [PATCH 14/25] Remove `.from_annot`. (#416) --- docs/source/changes.md | 1 + src/_pytask/collect_command.py | 11 ++---- src/_pytask/collect_utils.py | 39 +++++---------------- src/_pytask/execute.py | 2 -- src/_pytask/node_protocols.py | 5 ++- src/_pytask/nodes.py | 63 +++------------------------------- src/_pytask/profile.py | 2 -- src/_pytask/typing.py | 29 +--------------- tests/test_collect.py | 2 +- tests/test_collect_command.py | 9 ++--- tests/test_execute.py | 13 ++++--- tests/test_node_protocols.py | 6 ---- 12 files changed, 27 insertions(+), 155 deletions(-) diff --git a/docs/source/changes.md b/docs/source/changes.md index 11ebc8c20..0bc5a0621 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -32,6 +32,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`412` adds protocols for tasks. - {pull}`413` removes scripts to generate `.svg`s. - {pull}`414` allow more ruff rules. +- {pull}`416` removes `.from_annot` again. ## 0.3.2 - 2023-06-07 diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index bbbba4851..7ebeb56fa 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -29,7 +29,6 @@ from _pytask.pluginmanager import get_plugin_manager from _pytask.session import Session from _pytask.tree_util import tree_leaves -from _pytask.typing import no_value from rich.text import Text from rich.tree import Tree @@ -129,14 +128,10 @@ def _find_common_ancestor_of_all_nodes( all_paths.append(task.path) if show_nodes: all_paths.extend( - x.path - for x in tree_leaves(task.depends_on) - if isinstance(x, PPathNode) and x.path is not no_value + x.path for x in tree_leaves(task.depends_on) if isinstance(x, PPathNode) ) all_paths.extend( - x.path - for x in tree_leaves(task.produces) - if isinstance(x, PPathNode) and x.path is not no_value + x.path for x in tree_leaves(task.produces) if isinstance(x, PPathNode) ) return find_common_ancestor(*all_paths, *paths) @@ -208,7 +203,6 @@ def _print_collected_tasks( sorted_nodes = sorted(nodes, key=lambda x: x.name) for node in sorted_nodes: if isinstance(node, PPathNode): - assert node.path is not no_value if node.path.as_posix() in node.name: reduced_node_name = str( relative_to(node.path, common_ancestor) @@ -228,7 +222,6 @@ def _print_collected_tasks( tree_leaves(task.produces), key=lambda x: getattr(x, "path", x.name) ): if isinstance(node, PPathNode): - assert node.path is not no_value reduced_node_name = str(relative_to(node.path, common_ancestor)) url_style = create_url_style_for_path( node.path, editor_url_scheme diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 0a4767a20..b65805b41 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -1,7 +1,6 @@ """Contains utility functions for :mod:`_pytask.collect`.""" from __future__ import annotations -import functools import itertools import uuid import warnings @@ -275,17 +274,12 @@ def parse_dependencies_from_task_function( if parameter_name == "depends_on": continue - partialed_evolve = functools.partial( - _evolve_instance, - instance_from_annot=parameters_with_node_annot.get(parameter_name), - ) - nodes = tree_map_with_path( lambda p, x: _collect_dependency( session, path, name, - NodeInfo(parameter_name, p, partialed_evolve(x)), # noqa: B023 + NodeInfo(parameter_name, p, x), # noqa: B023 ), value, ) @@ -402,22 +396,21 @@ def parse_products_from_task_function( if parameters_with_product_annot: has_annotation = True - for parameter_name in parameters_with_product_annot: - if parameter_name in kwargs: - partialed_evolve = functools.partial( - _evolve_instance, - instance_from_annot=parameters_with_node_annot.get(parameter_name), - ) + for parameter_name in parameters_with_product_annot: + value = kwargs.get(parameter_name) or parameters_with_node_annot.get( + parameter_name + ) + if value: collected_products = tree_map_with_path( lambda p, x: _collect_product( session, path, name, - NodeInfo(parameter_name, p, partialed_evolve(x)), # noqa: B023 + NodeInfo(parameter_name, p, x), # noqa: B023 is_string_allowed=False, ), - kwargs[parameter_name], + value, ) out = {parameter_name: collected_products} @@ -602,19 +595,3 @@ def _collect_product( raise NodeNotCollectedError(msg) return collected_node - - -def _evolve_instance(x: Any, instance_from_annot: Node | None) -> Any: - """Evolve a value to a node if it is given by annotations.""" - if not instance_from_annot: - return x - - if not hasattr(instance_from_annot, "from_annot"): - msg = ( - f"The node {instance_from_annot!r} does not define '.from_annot' which is " - f"necessary to complete the node with the value {x!r}." - ) - raise AttributeError(msg) - - instance_from_annot.from_annot(x) # type: ignore[attr-defined] - return instance_from_annot diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index d20074e4b..902fe3526 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -36,7 +36,6 @@ from _pytask.tree_util import tree_leaves from _pytask.tree_util import tree_map from _pytask.tree_util import tree_structure -from _pytask.typing import no_value from rich.text import Text if TYPE_CHECKING: @@ -136,7 +135,6 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: for product in session.dag.successors(task.name): node = session.dag.nodes[product]["node"] if isinstance(node, PPathNode): - assert node.path is not no_value node.path.parent.mkdir(parents=True, exist_ok=True) would_be_executed = has_mark(task, "would_be_executed") diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py index dc014f5af..c2328b7b4 100644 --- a/src/_pytask/node_protocols.py +++ b/src/_pytask/node_protocols.py @@ -9,7 +9,6 @@ if TYPE_CHECKING: - from _pytask.typing import NoValue from _pytask.tree_util import PyTree from pathlib import Path from _pytask.mark import Mark @@ -54,7 +53,7 @@ class PPathNode(Node, Protocol): """ - path: Path | NoValue + path: Path @runtime_checkable @@ -82,4 +81,4 @@ class PTaskWithPath(PTask, Protocol): """ - path: Path | NoValue + path: Path diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index ab3a4298e..957b7ca60 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -3,7 +3,7 @@ import functools import hashlib -from pathlib import Path +from pathlib import Path # noqa: TCH003 from typing import Any from typing import Callable from typing import TYPE_CHECKING @@ -11,8 +11,6 @@ from _pytask.node_protocols import MetaNode from _pytask.node_protocols import Node from _pytask.node_protocols import PPathNode -from _pytask.typing import no_value -from _pytask.typing import NoValue from attrs import define from attrs import field @@ -76,33 +74,11 @@ def execute(self, **kwargs: Any) -> None: class PathNode(PPathNode): """The class for a node which is a path.""" - name: str = "" + name: str """Name of the node which makes it identifiable in the DAG.""" - path: Path | NoValue = no_value + path: Path """The path to the file.""" - def from_annot(self, value: Path) -> None: - """Set path and if other attributes are not set, set sensible defaults. - - Use it, if you want to control the name of the node. - - .. codeblock: python - - def task_example(value: Annotated[Any, PathNode(name="value")]): - ... - - - """ - if self.path is not no_value: - msg = "'path' is already defined." - raise ValueError(msg) - if not isinstance(value, Path): - msg = "'value' must be a 'pathlib.Path'." - raise TypeError(msg) - if not self.name: - self.name = value.as_posix() - self.path = value - @classmethod @functools.lru_cache def from_path(cls, path: Path) -> PathNode: @@ -122,24 +98,16 @@ def state(self) -> str | None: The state is given by the modification timestamp. """ - if self.path is no_value: - return None if self.path.exists(): return str(self.path.stat().st_mtime) return None def load(self) -> Path: """Load the value.""" - if self.path is no_value: - msg = "PathNode has no path defined." - raise ValueError(msg) return self.path def save(self, value: bytes | str) -> None: """Save strings or bytes to file.""" - if self.path is no_value: - msg = "'path' must be a 'pathlib.Path' to store data." - raise TypeError(msg) if isinstance(value, str): self.path.write_text(value) elif isinstance(value, bytes): @@ -155,40 +123,19 @@ class PythonNode(Node): name: str = "" """Name of the node.""" - value: Any | NoValue = no_value + value: Any | None = None """Value of the node.""" hash: bool | Callable[[Any], bool] = False # noqa: A003 """Whether the value should be hashed to determine the state.""" def load(self) -> Any: """Load the value.""" - if self.value is no_value: - msg = "PythonNode has no value defined." - raise ValueError(msg) return self.value def save(self, value: Any) -> None: """Save the value.""" self.value = value - def from_annot(self, value: Any) -> None: - """Set the value from a function annotation. - - Use it, if you want to add information on how a node handles an argument while - keeping the type of the value unrelated to pytask. For example, the node could - be hashed. - - .. codeblock: python - - def task_example(value: Annotated[Any, PythonNode(hash=True)]): - ... - - """ - if self.value is not no_value: - msg = "'value' is already defined." - raise ValueError(msg) - self.value = value - def state(self) -> str | None: """Calculate state of the node. @@ -204,8 +151,6 @@ def state(self) -> str | None: random integer and it would confuse users. """ - if self.value is no_value: - return None if self.hash: if callable(self.hash): return str(self.hash(self.value)) diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index e65c8fe0b..5fc2f7ced 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -29,7 +29,6 @@ from _pytask.pluginmanager import get_plugin_manager from _pytask.session import Session from _pytask.traceback import render_exc_info -from _pytask.typing import no_value from rich.table import Table from sqlalchemy import Column from sqlalchemy import Float @@ -230,7 +229,6 @@ def pytask_profile_add_info_on_task( node = session.dag.nodes[successor]["node"] if isinstance(node, PPathNode): with suppress(FileNotFoundError): - assert node.path is not no_value sum_bytes += node.path.stat().st_size profile[task.name]["Size of Products"] = _to_human_readable_size( diff --git a/src/_pytask/typing.py b/src/_pytask/typing.py index bcdb8dd01..e568c7eea 100644 --- a/src/_pytask/typing.py +++ b/src/_pytask/typing.py @@ -1,36 +1,9 @@ from __future__ import annotations -from enum import Enum -from typing import Literal - from attr import define -__all__ = ["no_value", "NoValue"] - - -class _NoValue(Enum): - """Sentinel value when no value was provided. - - For nodes, we need a different default value than :obj:`None` to signal that the - user provided no value. - - We make this an Enum - - - because it round-trips through pickle correctly (see GH#pandas/40397) - - because mypy does not understand singletons - - """ - - no_value = "NO_VALUE" - - def __repr__(self) -> str: - return "" - - -# Note: no_value is exported to the public API in pytask -no_value = _NoValue.no_value -NoValue = Literal[_NoValue.no_value] +__all__ = ["Product", "ProductType"] @define(frozen=True) diff --git a/tests/test_collect.py b/tests/test_collect.py index a462bf7d6..b0d3fcac0 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -433,7 +433,7 @@ def test_setting_name_for_path_node_via_annotation(tmp_path): from typing import Any def task_example( - path: Annotated[Path, Product, PathNode(name="product")] = Path("out.txt"), + path: Annotated[Path, Product, PathNode(path=Path("out.txt"), name="product")], ) -> None: path.write_text("text") """ diff --git a/tests/test_collect_command.py b/tests/test_collect_command.py index 1bb9fb345..86a483864 100644 --- a/tests/test_collect_command.py +++ b/tests/test_collect_command.py @@ -529,7 +529,6 @@ def state(self): def load(self): ... def save(self, value): ... - def from_annot(self, value): ... def task_example( data = CustomNode("custom", "text"), @@ -556,7 +555,6 @@ def test_node_protocol_for_custom_nodes_with_paths(runner, tmp_path): class PickleFile: name: str path: Path - value: Path def state(self): return str(self.path.stat().st_mtime) @@ -570,13 +568,10 @@ def save(self, value): with self.path.open("wb") as f: pickle.dump(value, f) - def from_annot(self, value): ... - - _PATH = Path(__file__).parent.joinpath("in.pkl") def task_example( - data = PickleFile(_PATH.as_posix(), _PATH, _PATH), + data = PickleFile(_PATH.as_posix(), _PATH), out: Annotated[Path, Product] = Path("out.txt"), ) -> None: out.write_text(data) @@ -598,7 +593,7 @@ def test_setting_name_for_python_node_via_annotation(runner, tmp_path): from typing import Any def task_example( - input: Annotated[str, PythonNode(name="node-name")] = "text", + input: Annotated[str, PythonNode(name="node-name", value="text")], path: Annotated[Path, Product] = Path("out.txt"), ) -> None: path.write_text(input) diff --git a/tests/test_execute.py b/tests/test_execute.py index c9f5b8d8b..0cb388f73 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -470,7 +470,7 @@ def task_example( "definition", [ " = PythonNode(value=data['dependency'], hash=True)", - ": Annotated[Any, PythonNode(hash=True)] = data['dependency']", + ": Annotated[Any, PythonNode(value=data['dependency'], hash=True)]", ], ) def test_task_with_hashed_python_node(runner, tmp_path, definition): @@ -532,8 +532,10 @@ def test_error_with_multiple_different_dep_annotations(runner, tmp_path): from pytask import Product, PythonNode, PathNode from typing import Any + annotation = Annotated[Any, PythonNode(), PathNode(name="a", path=Path("a.txt"))] + def task_example( - dependency: Annotated[Any, PythonNode(), PathNode()] = "hello", + dependency: annotation = "hello", path: Annotated[Path, Product] = Path("out.txt") ) -> None: path.write_text(dependency) @@ -615,9 +617,8 @@ def test_return_with_custom_type_annotation_as_return(runner, tmp_path): @attrs.define class PickleNode: - name: str = "" - path: Path | None = None - value: None = None + name: str + path: Path def state(self) -> str | None: if self.path.exists(): @@ -630,8 +631,6 @@ def load(self) -> Any: def save(self, value: Any) -> None: self.path.write_bytes(pickle.dumps(value)) - def from_annot(self, value: Any) -> None: ... - node = PickleNode("pickled_data", Path(__file__).parent.joinpath("data.pkl")) def task_example() -> Annotated[int, node]: diff --git a/tests/test_node_protocols.py b/tests/test_node_protocols.py index 02ce9867d..6fafd14fc 100644 --- a/tests/test_node_protocols.py +++ b/tests/test_node_protocols.py @@ -28,9 +28,6 @@ def load(self): def save(self, value): self.value = value - def from_annot(self, value): ... - - def task_example( data = CustomNode("custom", "text"), out: Annotated[Path, Product] = Path("out.txt"), @@ -70,9 +67,6 @@ def save(self, value): with self.path.open("wb") as f: pickle.dump(value, f) - def from_annot(self, value): ... - - _PATH = Path(__file__).parent.joinpath("in.pkl") def task_example( From 86c7a166d17354313d17bbded8cc7d41ee6351ef Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Sep 2023 00:16:31 +0200 Subject: [PATCH 15/25] Fix. --- src/_pytask/build.py | 4 +-- src/_pytask/collect.py | 24 ++++++++++++------ src/_pytask/console.py | 20 +++++++++------ src/_pytask/hookspecs.py | 3 +-- src/_pytask/node_protocols.py | 1 + src/_pytask/nodes.py | 46 +++++++++++++++++++++++++++++++++-- src/_pytask/report.py | 1 + tests/test_console.py | 4 +-- tests/test_execute.py | 2 +- 9 files changed, 82 insertions(+), 23 deletions(-) diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 8f2426472..97d268af0 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: - from _pytask.nodes import Task + from _pytask.node_protocols import PTask from typing import NoReturn @@ -66,7 +66,7 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 sort_table: bool = True, stop_after_first_failure: bool = False, strict_markers: bool = False, - tasks: Callable[..., Any] | Task | Iterable[Callable[..., Any] | Task] = (), + tasks: Callable[..., Any] | PTask | Iterable[Callable[..., Any] | PTask] = (), task_files: str | Iterable[str] = "task_*.py", trace: bool = False, verbose: int = 1, diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index bda842d5f..6d487a3ed 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -16,18 +16,20 @@ from _pytask.collect_utils import parse_products_from_task_function from _pytask.config import hookimpl from _pytask.config import IS_FILE_SYSTEM_CASE_SENSITIVE -from _pytask.console import _get_file from _pytask.console import console from _pytask.console import create_summary_panel from _pytask.console import format_task_name +from _pytask.console import get_file from _pytask.console import is_jupyter from _pytask.exceptions import CollectionError from _pytask.mark_utils import has_mark from _pytask.node_protocols import Node from _pytask.node_protocols import PTask +from _pytask.node_protocols import PTaskWithPath from _pytask.nodes import PathNode from _pytask.nodes import PythonNode from _pytask.nodes import Task +from _pytask.nodes import TempTask from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import count_outcomes from _pytask.path import find_case_sensitive_path @@ -55,7 +57,7 @@ def pytask_collect(session: Session) -> bool: session.tasks.extend( i.node for i in session.collection_reports - if i.outcome == CollectionOutcome.SUCCESS and isinstance(i.node, Task) + if i.outcome == CollectionOutcome.SUCCESS and isinstance(i.node, PTask) ) try: @@ -91,7 +93,7 @@ def _collect_from_paths(session: Session) -> None: def _collect_from_tasks(session: Session) -> None: """Collect tasks from user provided tasks via the functional interface.""" for raw_task in session.config.get("tasks", ()): - if isinstance(raw_task, Task): + if isinstance(raw_task, PTask): report = session.hook.pytask_collect_task_protocol( session=session, reports=session.collection_reports, @@ -107,7 +109,7 @@ def _collect_from_tasks(session: Session) -> None: if not hasattr(raw_task, "pytask_meta"): raw_task = task_decorator()(raw_task) # noqa: PLW2901 - path = _get_file(raw_task) + path = get_file(raw_task) if path.name == "": path = None @@ -210,7 +212,7 @@ def pytask_collect_task_protocol( @hookimpl(trylast=True) def pytask_collect_task( session: Session, path: Path, name: str, obj: Any -) -> Task | None: +) -> PTask | None: """Collect a task which is a function. There is some discussion on how to detect functions in this thread: @@ -229,6 +231,14 @@ def pytask_collect_task( # e.g. due to pytask_meta, in different layers of the wrapping. unwrapped = inspect.unwrap(obj) + if path is None: + return TempTask( + name=name, + function=unwrapped, + depends_on=dependencies, + produces=products, + markers=markers, + ) return Task( base_name=name, path=path, @@ -237,7 +247,7 @@ def pytask_collect_task( produces=products, markers=markers, ) - if isinstance(obj, Task): + if isinstance(obj, PTask): return obj return None @@ -326,7 +336,7 @@ def pytask_collect_modify_tasks(tasks: list[PTask]) -> None: """Given all tasks, assign a short uniquely identifiable name to each task.""" id_to_short_id = _find_shortest_uniquely_identifiable_name_for_tasks(tasks) for task in tasks: - if task.name in id_to_short_id and isinstance(task, Task): + if task.name in id_to_short_id and isinstance(task, PTaskWithPath): task.display_name = id_to_short_id[task.name] diff --git a/src/_pytask/console.py b/src/_pytask/console.py index bf0882ed5..47b021a2e 100644 --- a/src/_pytask/console.py +++ b/src/_pytask/console.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING import rich -from _pytask.nodes import Task +from _pytask.node_protocols import PTaskWithPath from rich.console import Console from rich.padding import Padding from rich.panel import Panel @@ -38,6 +38,8 @@ "console", "format_task_name", "format_strings_as_flat_tree", + "get_file", + "is_jupyter", "render_to_string", "unify_styles", ] @@ -149,7 +151,7 @@ def format_task_name(task: PTask, editor_url_scheme: str) -> Text: else: url_style = create_url_style_for_task(task.function, editor_url_scheme) - if isinstance(task, Task): + if isinstance(task, PTaskWithPath): path, task_name = task.display_name.split("::") task_id = Text.assemble( Text(path + "::", style="dim"), Text(task_name, style=url_style) @@ -179,7 +181,7 @@ def create_url_style_for_task( try: info = { - "path": _get_file(task_function), + "path": get_file(task_function), "line_number": _get_source_lines(task_function), } except (OSError, TypeError): @@ -200,7 +202,7 @@ def create_url_style_for_path(path: Path, edtior_url_scheme: str) -> Style: ) -def _get_file( +def get_file( function: Callable[..., Any], skipped_paths: list[Path] | None = None ) -> Path: """Get path to module where the function is defined. @@ -214,12 +216,12 @@ def _get_file( skipped_paths = _SKIPPED_PATHS if isinstance(function, functools.partial): - return _get_file(function.func) + return get_file(function.func) if ( hasattr(function, "__wrapped__") and Path(inspect.getsourcefile(function)) in skipped_paths ): - return _get_file(function.__wrapped__) + return get_file(function.__wrapped__) return Path(inspect.getsourcefile(function)) @@ -288,7 +290,11 @@ def create_summary_panel( def is_jupyter() -> bool: # pragma: no cover - """Check if we're running in a Jupyter notebook.""" + """Check if we're running in a Jupyter notebook. + + Copied from rich. + + """ try: get_ipython # type: ignore[name-defined] # noqa: B018 except NameError: diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index 294ae929a..ae7ecf3f9 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -21,7 +21,6 @@ import networkx as nx import pathlib from _pytask.session import Session - from _pytask.nodes import Task from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome from _pytask.reports import CollectionReport @@ -182,7 +181,7 @@ def pytask_collect_task_setup( @hookspec(firstresult=True) def pytask_collect_task( session: Session, path: pathlib.Path, name: str, obj: Any -) -> Task: +) -> PTask: """Collect a single task.""" diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py index c2328b7b4..54a52bd68 100644 --- a/src/_pytask/node_protocols.py +++ b/src/_pytask/node_protocols.py @@ -61,6 +61,7 @@ class PTask(MetaNode, Protocol): """Protocol for nodes.""" name: str + display_name: str depends_on: PyTree[Node] produces: PyTree[Node] markers: list[Mark] diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index 957b7ca60..c8cf215e2 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -3,14 +3,16 @@ import functools import hashlib +import inspect from pathlib import Path # noqa: TCH003 from typing import Any from typing import Callable from typing import TYPE_CHECKING -from _pytask.node_protocols import MetaNode from _pytask.node_protocols import Node from _pytask.node_protocols import PPathNode +from _pytask.node_protocols import PTask +from _pytask.node_protocols import PTaskWithPath from attrs import define from attrs import field @@ -24,7 +26,47 @@ @define(kw_only=True) -class Task(MetaNode): +class TempTask(PTask): + """The class for tasks which are Python functions.""" + + name: str + """The base name of the task.""" + function: Callable[..., Any] + """The task function.""" + display_name: str | None = field(default=None, init=False) + """The shortest uniquely identifiable name for task for display.""" + depends_on: PyTree[Node] = field(factory=dict) + """A list of dependencies of task.""" + produces: PyTree[Node] = field(factory=dict) + """A list of products of task.""" + markers: list[Mark] = field(factory=list) + """A list of markers attached to the task function.""" + report_sections: list[tuple[str, str, str]] = field(factory=list) + """Reports with entries for when, what, and content.""" + attributes: dict[Any, Any] = field(factory=dict) + """A dictionary to store additional information of the task.""" + + def __attrs_post_init__(self: TempTask) -> None: + """Change class after initialization.""" + if self.display_name is None: + self.display_name = self.name + + def state(self) -> str | None: + """Return the state of the node.""" + try: + source = inspect.getsource(self.function) + except OSError: + return None + else: + return hashlib.sha256(source.encode()).hexdigest() + + def execute(self, **kwargs: Any) -> None: + """Execute the task.""" + return self.function(**kwargs) + + +@define(kw_only=True) +class Task(PTaskWithPath): """The class for tasks which are Python functions.""" base_name: str diff --git a/src/_pytask/report.py b/src/_pytask/report.py index 74e524eca..c29cdec42 100644 --- a/src/_pytask/report.py +++ b/src/_pytask/report.py @@ -37,6 +37,7 @@ def from_exception( exc_info: ExceptionInfo, node: MetaNode | None = None, ) -> CollectionReport: + exc_info = remove_internal_traceback_frames_from_exc_info(exc_info) return cls(outcome=outcome, node=node, exc_info=exc_info) diff --git a/tests/test_console.py b/tests/test_console.py index f4ba00485..22b61b845 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -4,12 +4,12 @@ from pathlib import Path import pytest -from _pytask.console import _get_file from _pytask.console import _get_source_lines from _pytask.console import create_summary_panel from _pytask.console import create_url_style_for_path from _pytask.console import create_url_style_for_task from _pytask.console import format_task_name +from _pytask.console import get_file from _pytask.console import render_to_string from pytask import CollectionOutcome from pytask import console @@ -163,7 +163,7 @@ def test_format_task_id( ], ) def test_get_file(task_func, skipped_paths, expected): - result = _get_file(task_func, skipped_paths) + result = get_file(task_func, skipped_paths) assert result == expected diff --git a/tests/test_execute.py b/tests/test_execute.py index 0cb388f73..846184750 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -734,7 +734,7 @@ def task_create_file(text: Annotated[int, node_text]) -> Annotated[str, node_fil @pytest.mark.xfail(sys.platform == "win32", reason="Decoding issues in Gitlab Actions.") def test_execute_tasks_via_functional_api(tmp_path): source = """ - from _pytask.nodes import PathNode + from pytask import PathNode import pytask from pytask import PythonNode from typing_extensions import Annotated From a4dd4c52d3185fc883ab63f75f1a29a3f2f712d7 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Sep 2023 00:36:07 +0200 Subject: [PATCH 16/25] Set path to None for Jupyter. --- src/_pytask/collect.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 6d487a3ed..04d40e094 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -113,7 +113,8 @@ def _collect_from_tasks(session: Session) -> None: if path.name == "": path = None - if is_jupyter(): + # Detect whether a path is defined in a Jupyter notebook. + if is_jupyter() and "ipykernel" in path.as_posix() and path.suffix == ".py": path = None name = raw_task.pytask_meta.name From bf6f9ba7ecc09bf1a095b16c243fc1b3e984ceab Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Sep 2023 10:27:52 +0200 Subject: [PATCH 17/25] Add docstring to build. --- src/_pytask/build.py | 78 +++++++++++++++++++++++++++++++++++++++--- src/_pytask/capture.py | 5 ++- src/pytask/__init__.py | 2 +- 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/src/_pytask/build.py b/src/_pytask/build.py index 97d268af0..15e1e0719 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -6,6 +6,7 @@ from typing import Any from typing import Callable from typing import Iterable +from typing import Literal from typing import TYPE_CHECKING import click @@ -41,14 +42,15 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: def build( # noqa: C901, PLR0912, PLR0913, PLR0915 *, - capture: CaptureMethod = CaptureMethod.NO, + capture: Literal["fd", "no", "sys", "tee-sys"] | CaptureMethod = CaptureMethod.NO, check_casing_of_paths: bool = True, config: Path | None = None, database_url: str = "", debug_pytask: bool = False, disable_warnings: bool = False, dry_run: bool = False, - editor_url_scheme: str = "file", + editor_url_scheme: Literal["no_link", "file", "vscode", "pycharm"] # noqa: PYI051 + | str = "file", expression: str = "", force: bool = False, ignore: Iterable[str] = (), @@ -78,9 +80,75 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 command line interface. It can also be used to run pytask interactively. Pass configuration in a dictionary. + Parameters + ---------- + capture + The capture method for stdout and stderr. + check_casing_of_paths + Whether errors should be raised when file names have different casings. + config + A path to the configuration file. + database_url + An URL to the database that tracks the status of tasks. + debug_pytask + Whether debug information should be shown. + disable_warnings + Whether warnings should be disabled and not displayed. + dry_run + Whether a dry-run should be performed that shows which tasks need to be rerun. + editor_url_scheme + An url scheme that allows to click on task names, node names and filenames and + jump right into you preferred edior to the right line. + expression + Same as ``-k`` on the command line. Select tasks via expressions on task ids. + force + Run tasks even though they would be skipped since nothing has changed. + ignore + A pattern to ignore files or directories. Refer to ``pathlib.Path.match`` + for more info. + marker_expression + Same as ``-m`` on the command line. Select tasks via marker expressions. + max_failures + Stop after some failures. + n_entries_in_table + How many entries to display in the table during the execution. Tasks which are + running are always displayed. + paths + A path or collection of paths where pytask looks for the configuration and + tasks. + pdb + Start the interactive debugger on errors. + pdb_cls + Start a custom debugger on errors. For example: + ``--pdbcls=IPython.terminal.debugger:TerminalPdb`` + s + Shortcut for ``pytask.build(capture"no")``. + show_capture + Choose which captured output should be shown for failed tasks. + show_errors_immediately + Show errors with tracebacks as soon as the task fails. + show_locals + Show local variables in tracebacks. + show_traceback + Choose whether tracebacks should be displayed or not. + sort_table + Sort the table of tasks at the end of the execution. + stop_after_first_failure + Stop after the first failure. + strict_markers + Raise errors for unknown markers. + tasks + A task or a collection of tasks that is passed to ``pytask.build(tasks=...)``. + task_files + A pattern to describe modules that contain tasks. + trace + Enter debugger in the beginning of each task. + verbose + Make pytask verbose (>= 0) or quiet (= 0). + Returns ------- - session : _pytask.session.Session + session : pytask.Session The session captures all the information of the current run. """ @@ -224,13 +292,13 @@ def build( # noqa: C901, PLR0912, PLR0913, PLR0915 "--show-errors-immediately", is_flag=True, default=False, - help="Print errors with tracebacks as soon as the task fails.", + help="Show errors with tracebacks as soon as the task fails.", ) @click.option( "--show-traceback/--show-no-traceback", type=bool, default=True, - help=("Choose whether tracebacks should be displayed or not."), + help="Choose whether tracebacks should be displayed or not.", ) @click.option( "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry-run." diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index 8ba531c85..11c42365a 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -77,7 +77,7 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None: ["--show-capture"], type=EnumChoice(ShowCapture), default=ShowCapture.ALL, - help=("Choose which captured output should be shown for failed tasks."), + help="Choose which captured output should be shown for failed tasks.", ), ] cli.commands["build"].params.extend(additional_parameters) @@ -90,6 +90,9 @@ def pytask_parse_config(config: dict[str, Any]) -> None: Note that, ``-s`` is a shortcut for ``--capture=no``. """ + if isinstance(config["capture"], str): + config["capture"] = CaptureMethod(config["capture"]) + if config["s"]: config["capture"] = CaptureMethod.NO diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py index 2822677ed..28ec1e102 100644 --- a/src/pytask/__init__.py +++ b/src/pytask/__init__.py @@ -108,6 +108,7 @@ "TaskOutcome", "WarningReport", "__version__", + "build", "build_dag", "check_for_optional_program", "cli", @@ -121,7 +122,6 @@ "has_mark", "hookimpl", "import_optional_dependency", - "build", "mark", "parse_nodes", "parse_warning_filter", From 77f39cf163f6bd50f7546c0ecfa494058bc5e0b2 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Sep 2023 10:57:58 +0200 Subject: [PATCH 18/25] Rename task. --- src/_pytask/collect.py | 4 ++-- src/_pytask/nodes.py | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 04d40e094..0dda36945 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -29,7 +29,7 @@ from _pytask.nodes import PathNode from _pytask.nodes import PythonNode from _pytask.nodes import Task -from _pytask.nodes import TempTask +from _pytask.nodes import TaskWithoutSource from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import count_outcomes from _pytask.path import find_case_sensitive_path @@ -233,7 +233,7 @@ def pytask_collect_task( unwrapped = inspect.unwrap(obj) if path is None: - return TempTask( + return TaskWithoutSource( name=name, function=unwrapped, depends_on=dependencies, diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index c8cf215e2..354b3988b 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -1,4 +1,4 @@ -"""Deals with nodes which are dependencies or products of a task.""" +"""Contains implementations of tasks and nodes following the node protocols.""" from __future__ import annotations import functools @@ -26,8 +26,14 @@ @define(kw_only=True) -class TempTask(PTask): - """The class for tasks which are Python functions.""" +class TaskWithoutSource(PTask): + """The class for tasks without a source file. + + Tasks may have no source file because + - they are dynamically created in a REPL. + - they are created in a Jupyter notebook. + + """ name: str """The base name of the task.""" @@ -46,7 +52,7 @@ class TempTask(PTask): attributes: dict[Any, Any] = field(factory=dict) """A dictionary to store additional information of the task.""" - def __attrs_post_init__(self: TempTask) -> None: + def __attrs_post_init__(self: TaskWithoutSource) -> None: """Change class after initialization.""" if self.display_name is None: self.display_name = self.name From cce15d8efead2339ceeab798f7642894f52f1781 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Sep 2023 11:01:28 +0200 Subject: [PATCH 19/25] Rename task. --- src/_pytask/collect.py | 4 ++-- src/_pytask/nodes.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 0dda36945..2c92e02f8 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -29,7 +29,7 @@ from _pytask.nodes import PathNode from _pytask.nodes import PythonNode from _pytask.nodes import Task -from _pytask.nodes import TaskWithoutSource +from _pytask.nodes import TaskWithoutPath from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import count_outcomes from _pytask.path import find_case_sensitive_path @@ -233,7 +233,7 @@ def pytask_collect_task( unwrapped = inspect.unwrap(obj) if path is None: - return TaskWithoutSource( + return TaskWithoutPath( name=name, function=unwrapped, depends_on=dependencies, diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index 354b3988b..32d763a85 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -26,7 +26,7 @@ @define(kw_only=True) -class TaskWithoutSource(PTask): +class TaskWithoutPath(PTask): """The class for tasks without a source file. Tasks may have no source file because @@ -52,7 +52,7 @@ class TaskWithoutSource(PTask): attributes: dict[Any, Any] = field(factory=dict) """A dictionary to store additional information of the task.""" - def __attrs_post_init__(self: TaskWithoutSource) -> None: + def __attrs_post_init__(self: TaskWithoutPath) -> None: """Change class after initialization.""" if self.display_name is None: self.display_name = self.name From ae81ea430070c69ed8c935f4bb52f5718d06a8c2 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Sep 2023 12:01:51 +0200 Subject: [PATCH 20/25] Add error when nodes are supplied via kwargs and annotations. --- src/_pytask/collect_utils.py | 53 ++++++++++++++++++++++++++---------- tests/test_collect.py | 42 ++++++++++++++++++++++++++++ tests/test_task.py | 4 +-- 3 files changed, 81 insertions(+), 18 deletions(-) diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index b65805b41..55f5aefa9 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -261,8 +261,14 @@ def parse_dependencies_from_task_function( # Complete kwargs with node annotations, when no value is given by kwargs. for name in list(parameters_with_node_annot): if name not in kwargs: - kwargs[name] = parameters_with_node_annot[name] - parameters_with_node_annot.pop(name) + kwargs[name] = parameters_with_node_annot.pop(name) + else: + msg = ( + f"The value for the parameter {name!r} is defined twice in " + "'@pytask.mark.task(kwargs=...)' and in the type annotation. Choose " + "only one option." + ) + raise ValueError(msg) for parameter_name, value in kwargs.items(): if ( @@ -344,7 +350,7 @@ def task_example(produces: Annotated[..., Product]): """ -def parse_products_from_task_function( +def parse_products_from_task_function( # noqa: C901 session: Session, path: Path, name: str, obj: Any ) -> dict[str, Any]: """Parse products from task function. @@ -398,21 +404,38 @@ def parse_products_from_task_function( has_annotation = True for parameter_name in parameters_with_product_annot: + if ( + parameter_name not in kwargs + and parameter_name not in parameters_with_node_annot + ): + continue + + if ( + parameter_name in kwargs + and parameter_name in parameters_with_node_annot + ): + msg = ( + f"The value for the parameter {name!r} is defined twice in " + "'@pytask.mark.task(kwargs=...)' and in the type annotation. " + "Choose only one option." + ) + raise ValueError(msg) + value = kwargs.get(parameter_name) or parameters_with_node_annot.get( parameter_name ) - if value: - collected_products = tree_map_with_path( - lambda p, x: _collect_product( - session, - path, - name, - NodeInfo(parameter_name, p, x), # noqa: B023 - is_string_allowed=False, - ), - value, - ) - out = {parameter_name: collected_products} + + collected_products = tree_map_with_path( + lambda p, x: _collect_product( + session, + path, + name, + NodeInfo(parameter_name, p, x), # noqa: B023 + is_string_allowed=False, + ), + value, + ) + out = {parameter_name: collected_products} if "return" in parameters_with_node_annot: has_return = True diff --git a/tests/test_collect.py b/tests/test_collect.py index b0d3fcac0..f4aa1e9d8 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -443,3 +443,45 @@ def task_example( assert session.exit_code == ExitCode.OK product = session.tasks[0].produces["path"] assert product.name == "product" + + +@pytest.mark.end_to_end() +def test_error_when_dependency_is_defined_in_kwargs_and_annotation(runner, tmp_path): + source = """ + import pytask + from pathlib import Path + from typing_extensions import Annotated + from pytask import Product, PathNode + from pytask import PythonNode + + @pytask.mark.task(kwargs={"in_": "world"}) + def task_example( + in_: Annotated[str, PythonNode(name="string", value="hello")], + path: Annotated[Path, Product, PathNode(path=Path("out.txt"), name="product")], + ) -> None: + path.write_text(in_) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert "ValueError: The value for the parameter 'in_'" in result.output + + +@pytest.mark.end_to_end() +def test_error_when_product_is_defined_in_kwargs_and_annotation(runner, tmp_path): + source = """ + import pytask + from pathlib import Path + from typing_extensions import Annotated + from pytask import Product, PathNode + + node = PathNode(path=Path("out.txt"), name="product") + + @pytask.mark.task(kwargs={"path": node}) + def task_example(path: Annotated[Path, Product, node]) -> None: + path.write_text("text") + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert "ValueError: The value for the parameter 'path'" in result.output diff --git a/tests/test_task.py b/tests/test_task.py index 0c71a5f63..228e1e3cc 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -471,9 +471,7 @@ class Args(NamedTuple): @pytask.mark.task(kwargs=args) def task_example( - path_in: Path, - arg: Annotated[str, PythonNode(hash=True)], - path_out: Annotated[Path, Product] + path_in: Path, arg: str, path_out: Annotated[Path, Product] ) -> None: path_out.write_text(path_in.read_text() + " " + arg) """ From 16e804b9de9c5259b1e5c85c0038e08f68981326 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Sep 2023 12:45:31 +0200 Subject: [PATCH 21/25] Add jupyter test. --- .gitignore | 2 + .pre-commit-config.yaml | 4 ++ environment.yml | 1 + .../test_functional_interface.ipynb | 71 +++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 tests/test_jupyter/test_functional_interface.ipynb diff --git a/.gitignore b/.gitignore index 3fbda029b..b08979dfe 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ build dist src/_pytask/_version.py *.pkl + +tests/test_jupyter/*.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d163715c..aec3505f1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -100,6 +100,10 @@ repos: docs/source/tutorials/selecting_tasks.md| docs/source/tutorials/set_up_a_project.md )$ +- repo: https://github.com/kynan/nbstripout + rev: 0.6.1 + hooks: + - id: nbstripout - repo: https://github.com/codespell-project/codespell rev: v2.2.5 hooks: diff --git a/environment.yml b/environment.yml index 528ad2798..d5e671a1a 100644 --- a/environment.yml +++ b/environment.yml @@ -26,6 +26,7 @@ dependencies: - black - jupyterlab - matplotlib + - nbval - pre-commit - pygraphviz - pytest diff --git a/tests/test_jupyter/test_functional_interface.ipynb b/tests/test_jupyter/test_functional_interface.ipynb new file mode 100644 index 000000000..2b7be985e --- /dev/null +++ b/tests/test_jupyter/test_functional_interface.ipynb @@ -0,0 +1,71 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "12bc75b1", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "import pytask\n", + "\n", + "from pytask import ExitCode\n", + "from pytask import PathNode\n", + "from pytask import PythonNode\n", + "from typing_extensions import Annotated" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29ac7311", + "metadata": {}, + "outputs": [], + "source": [ + "node_text = PythonNode(name=\"text\", hash=True)\n", + "\n", + "def create_text() -> Annotated[int, node_text]:\n", + " return \"This is the text.\"\n", + "\n", + "node_file = PathNode.from_path(Path(\"file.txt\").resolve())\n", + "\n", + "def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]:\n", + " return text" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "738c9418", + "metadata": {}, + "outputs": [], + "source": [ + "session = pytask.build(tasks=[create_file, create_text])\n", + "assert session.exit_code == ExitCode.OK" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From d9d1fdbe2214f657f0405ab398ece90ba30030a0 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Sep 2023 17:51:12 +0200 Subject: [PATCH 22/25] Simplify nodes, fix resolving of paths. --- src/_pytask/collect.py | 78 +++++++++++++++++++++++------------ src/_pytask/collect_utils.py | 12 +++--- src/_pytask/console.py | 4 +- src/_pytask/dag.py | 4 +- src/_pytask/hookspecs.py | 20 ++++----- src/_pytask/node_protocols.py | 14 ++++--- src/_pytask/nodes.py | 23 ++++------- src/_pytask/parameters.py | 2 +- src/pytask/__init__.py | 8 ++++ tests/test_collect.py | 21 +++++++++- 10 files changed, 117 insertions(+), 69 deletions(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 2c92e02f8..a8c857bd3 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -23,9 +23,9 @@ from _pytask.console import is_jupyter from _pytask.exceptions import CollectionError from _pytask.mark_utils import has_mark -from _pytask.node_protocols import Node +from _pytask.node_protocols import PNode +from _pytask.node_protocols import PPathNode from _pytask.node_protocols import PTask -from _pytask.node_protocols import PTaskWithPath from _pytask.nodes import PathNode from _pytask.nodes import PythonNode from _pytask.nodes import Task @@ -253,21 +253,30 @@ def pytask_collect_task( return None -_TEMPLATE_ERROR: str = ( - "The provided path of the dependency/product in the marker is\n\n{}\n\n, but the " - "path of the file on disk is\n\n{}\n\nCase-sensitive file systems would raise an " - "error because the upper and lower case format of the paths does not match.\n\n" - "Please, align the names to ensure reproducibility on case-sensitive file systems " - "(often Linux or macOS) or disable this error with 'check_casing_of_paths = false' " - " in your pytask configuration file.\n\n" - "Hint: If parts of the path preceding your project directory are not properly " - "formatted, check whether you need to call `.resolve()` on `SRC`, `BLD` or other " - "paths created from the `__file__` attribute of a module." -) +_TEMPLATE_ERROR: str = """\ +The provided path of the dependency/product is + +{} + +, but the path of the file on disk is + +{} + +Case-sensitive file systems would raise an error because the upper and lower case \ +format of the paths does not match. + +Please, align the names to ensure reproducibility on case-sensitive file systems \ +(often Linux or macOS) or disable this error with 'check_casing_of_paths = false' in \ +your pytask configuration file. + +Hint: If parts of the path preceding your project directory are not properly \ +formatted, check whether you need to call `.resolve()` on `SRC`, `BLD` or other paths \ +created from the `__file__` attribute of a module. +""" @hookimpl(trylast=True) -def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> Node: +def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> PNode: """Collect a node of a task as a :class:`pytask.nodes.PathNode`. Strings are assumed to be paths. This might be a strict assumption, but since this @@ -286,7 +295,17 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> No node.name = node_info.arg_name + suffix return node - if isinstance(node, Node): + if isinstance(node, PPathNode) and not node.path.is_absolute(): + node.path = path.parent.joinpath(node.path) + + # ``normpath`` removes ``../`` from the path which is necessary for the casing + # check which will fail since ``.resolves()`` also normalizes a path. + node.path = Path(os.path.normpath(node.path)) + _raise_error_if_casing_of_path_is_wrong( + node.path, session.config["check_casing_of_paths"] + ) + + if isinstance(node, PNode): return node if isinstance(node, Path): @@ -296,16 +315,9 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> No # ``normpath`` removes ``../`` from the path which is necessary for the casing # check which will fail since ``.resolves()`` also normalizes a path. node = Path(os.path.normpath(node)) - - if ( - not IS_FILE_SYSTEM_CASE_SENSITIVE - and session.config["check_casing_of_paths"] - and sys.platform == "win32" - ): - case_sensitive_path = find_case_sensitive_path(node, "win32") - if str(node) != str(case_sensitive_path): - raise ValueError(_TEMPLATE_ERROR.format(node, case_sensitive_path)) - + _raise_error_if_casing_of_path_is_wrong( + node, session.config["check_casing_of_paths"] + ) return PathNode.from_path(node) suffix = "-" + "-".join(map(str, node_info.path)) if node_info.path else "" @@ -313,6 +325,20 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> No return PythonNode(value=node, name=node_name) +def _raise_error_if_casing_of_path_is_wrong( + path: Path, check_casing_of_paths: bool +) -> None: + """Raise an error if the path does not have the correct casing.""" + if ( + not IS_FILE_SYSTEM_CASE_SENSITIVE + and sys.platform == "win32" + and check_casing_of_paths + ): + case_sensitive_path = find_case_sensitive_path(path, "win32") + if str(path) != str(case_sensitive_path): + raise ValueError(_TEMPLATE_ERROR.format(path, case_sensitive_path)) + + def _not_ignored_paths( paths: Iterable[Path], session: Session ) -> Generator[Path, None, None]: @@ -337,7 +363,7 @@ def pytask_collect_modify_tasks(tasks: list[PTask]) -> None: """Given all tasks, assign a short uniquely identifiable name to each task.""" id_to_short_id = _find_shortest_uniquely_identifiable_name_for_tasks(tasks) for task in tasks: - if task.name in id_to_short_id and isinstance(task, PTaskWithPath): + if task.name in id_to_short_id and isinstance(task, Task): task.display_name = id_to_short_id[task.name] diff --git a/src/_pytask/collect_utils.py b/src/_pytask/collect_utils.py index 55f5aefa9..de138d39a 100644 --- a/src/_pytask/collect_utils.py +++ b/src/_pytask/collect_utils.py @@ -16,7 +16,7 @@ from _pytask.mark_utils import has_mark from _pytask.mark_utils import remove_marks from _pytask.models import NodeInfo -from _pytask.node_protocols import Node +from _pytask.node_protocols import PNode from _pytask.nodes import PythonNode from _pytask.shared import find_duplicates from _pytask.task_utils import parse_keyword_arguments_from_signature_defaults @@ -295,14 +295,14 @@ def parse_dependencies_from_task_function( are_all_nodes_python_nodes_without_hash = all( isinstance(x, PythonNode) and not x.hash for x in tree_leaves(nodes) ) - if not isinstance(nodes, Node) and are_all_nodes_python_nodes_without_hash: + if not isinstance(nodes, PNode) and are_all_nodes_python_nodes_without_hash: dependencies[parameter_name] = PythonNode(value=value, name=parameter_name) else: dependencies[parameter_name] = nodes return dependencies -def _find_args_with_node_annotation(func: Callable[..., Any]) -> dict[str, Node]: +def _find_args_with_node_annotation(func: Callable[..., Any]) -> dict[str, PNode]: """Find args with node annotations.""" annotations = get_annotations(func, eval_str=True) metas = { @@ -515,7 +515,7 @@ def _find_args_with_product_annotation(func: Callable[..., Any]) -> list[str]: def _collect_decorator_node( session: Session, path: Path, name: str, node_info: NodeInfo -) -> Node: +) -> PNode: """Collect nodes for a task. Raises @@ -553,7 +553,7 @@ def _collect_decorator_node( def _collect_dependency( session: Session, path: Path, name: str, node_info: NodeInfo -) -> Node: +) -> PNode: """Collect nodes for a task. Raises @@ -581,7 +581,7 @@ def _collect_product( task_name: str, node_info: NodeInfo, is_string_allowed: bool = False, -) -> Node: +) -> PNode: """Collect products for a task. Defining products with strings is only allowed when using the decorator. Parameter diff --git a/src/_pytask/console.py b/src/_pytask/console.py index 47b021a2e..5bbea462d 100644 --- a/src/_pytask/console.py +++ b/src/_pytask/console.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING import rich -from _pytask.node_protocols import PTaskWithPath +from _pytask.nodes import Task from rich.console import Console from rich.padding import Padding from rich.panel import Panel @@ -151,7 +151,7 @@ def format_task_name(task: PTask, editor_url_scheme: str) -> Text: else: url_style = create_url_style_for_task(task.function, editor_url_scheme) - if isinstance(task, PTaskWithPath): + if isinstance(task, Task): path, task_name = task.display_name.split("::") task_id = Text.assemble( Text(path + "::", style="dim"), Text(task_name, style=url_style) diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py index fbc3cd665..1dfde8d6b 100644 --- a/src/_pytask/dag.py +++ b/src/_pytask/dag.py @@ -24,7 +24,7 @@ from _pytask.mark_utils import get_marks from _pytask.mark_utils import has_mark from _pytask.node_protocols import MetaNode -from _pytask.node_protocols import Node +from _pytask.node_protocols import PNode from _pytask.node_protocols import PPathNode from _pytask.node_protocols import PTask from _pytask.node_protocols import PTaskWithPath @@ -242,7 +242,7 @@ def _check_if_root_nodes_are_available(dag: nx.DiGraph) -> None: def _check_if_tasks_are_skipped( - node: Node, dag: nx.DiGraph, is_task_skipped: dict[str, bool] + node: PNode, dag: nx.DiGraph, is_task_skipped: dict[str, bool] ) -> tuple[bool, dict[str, bool]]: """Check for a given node whether it is only used by skipped tasks.""" are_all_tasks_skipped = [] diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index ae7ecf3f9..77391aa4f 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -15,11 +15,11 @@ if TYPE_CHECKING: from _pytask.node_protocols import MetaNode from _pytask.models import NodeInfo - from _pytask.node_protocols import Node + from _pytask.node_protocols import PNode import click from _pytask.node_protocols import PTask import networkx as nx - import pathlib + from pathlib import Path from _pytask.session import Session from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome @@ -118,7 +118,7 @@ def pytask_collect(session: Session) -> Any: @hookspec(firstresult=True) -def pytask_ignore_collect(path: pathlib.Path, config: dict[str, Any]) -> bool: +def pytask_ignore_collect(path: Path, config: dict[str, Any]) -> bool: """Ignore collected path. This hook is indicates for each directory and file whether it should be ignored. @@ -138,7 +138,7 @@ def pytask_collect_modify_tasks(session: Session, tasks: list[PTask]) -> None: @hookspec(firstresult=True) def pytask_collect_file_protocol( - session: Session, path: pathlib.Path, reports: list[CollectionReport] + session: Session, path: Path, reports: list[CollectionReport] ) -> list[CollectionReport]: """Start protocol to collect files. @@ -150,7 +150,7 @@ def pytask_collect_file_protocol( @hookspec def pytask_collect_file( - session: Session, path: pathlib.Path, reports: list[CollectionReport] + session: Session, path: Path, reports: list[CollectionReport] ) -> list[CollectionReport] | None: """Collect tasks from a file. @@ -166,21 +166,21 @@ def pytask_collect_file_log(session: Session, reports: list[CollectionReport]) - @hookspec(firstresult=True) def pytask_collect_task_protocol( - session: Session, path: pathlib.Path, name: str, obj: Any + session: Session, path: Path | None, name: str, obj: Any ) -> CollectionReport | None: """Start protocol to collect tasks.""" @hookspec def pytask_collect_task_setup( - session: Session, path: pathlib.Path, name: str, obj: Any + session: Session, path: Path | None, name: str, obj: Any ) -> None: """Steps before collecting a task.""" @hookspec(firstresult=True) def pytask_collect_task( - session: Session, path: pathlib.Path, name: str, obj: Any + session: Session, path: Path | None, name: str, obj: Any ) -> PTask: """Collect a single task.""" @@ -196,8 +196,8 @@ def pytask_collect_task_teardown(session: Session, task: PTask) -> None: @hookspec(firstresult=True) def pytask_collect_node( - session: Session, path: pathlib.Path, node_info: NodeInfo -) -> Node | None: + session: Session, path: Path, node_info: NodeInfo +) -> PNode | None: """Collect a node which is a dependency or a product of a task.""" diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py index 54a52bd68..8eee22c94 100644 --- a/src/_pytask/node_protocols.py +++ b/src/_pytask/node_protocols.py @@ -14,11 +14,14 @@ from _pytask.mark import Mark +__all__ = ["MetaNode", "PNode", "PPathNode", "PTask", "PTaskWithPath"] + + @runtime_checkable class MetaNode(Protocol): """Protocol for an intersection between nodes and tasks.""" - name: str | None + name: str """The name of node that must be unique.""" @abstractmethod @@ -33,7 +36,7 @@ def state(self) -> str | None: @runtime_checkable -class Node(MetaNode, Protocol): +class PNode(MetaNode, Protocol): """Protocol for nodes.""" def load(self) -> Any: @@ -46,7 +49,7 @@ def save(self, value: Any) -> Any: @runtime_checkable -class PPathNode(Node, Protocol): +class PPathNode(PNode, Protocol): """Nodes with paths. Nodes with paths receive special handling when it comes to printing their names. @@ -61,9 +64,8 @@ class PTask(MetaNode, Protocol): """Protocol for nodes.""" name: str - display_name: str - depends_on: PyTree[Node] - produces: PyTree[Node] + depends_on: PyTree[PNode] + produces: PyTree[PNode] markers: list[Mark] report_sections: list[tuple[str, str, str]] attributes: dict[Any, Any] diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index 32d763a85..a82f42a1f 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -9,7 +9,7 @@ from typing import Callable from typing import TYPE_CHECKING -from _pytask.node_protocols import Node +from _pytask.node_protocols import PNode from _pytask.node_protocols import PPathNode from _pytask.node_protocols import PTask from _pytask.node_protocols import PTaskWithPath @@ -22,7 +22,7 @@ from _pytask.mark import Mark -__all__ = ["PathNode", "PythonNode", "Task"] +__all__ = ["PathNode", "PythonNode", "Task", "TaskWithoutPath"] @define(kw_only=True) @@ -39,11 +39,9 @@ class TaskWithoutPath(PTask): """The base name of the task.""" function: Callable[..., Any] """The task function.""" - display_name: str | None = field(default=None, init=False) - """The shortest uniquely identifiable name for task for display.""" - depends_on: PyTree[Node] = field(factory=dict) + depends_on: PyTree[PNode] = field(factory=dict) """A list of dependencies of task.""" - produces: PyTree[Node] = field(factory=dict) + produces: PyTree[PNode] = field(factory=dict) """A list of products of task.""" markers: list[Mark] = field(factory=list) """A list of markers attached to the task function.""" @@ -52,11 +50,6 @@ class TaskWithoutPath(PTask): attributes: dict[Any, Any] = field(factory=dict) """A dictionary to store additional information of the task.""" - def __attrs_post_init__(self: TaskWithoutPath) -> None: - """Change class after initialization.""" - if self.display_name is None: - self.display_name = self.name - def state(self) -> str | None: """Return the state of the node.""" try: @@ -85,9 +78,9 @@ class Task(PTaskWithPath): """The name of the task.""" display_name: str | None = field(default=None, init=False) """The shortest uniquely identifiable name for task for display.""" - depends_on: PyTree[Node] = field(factory=dict) + depends_on: PyTree[PNode] = field(factory=dict) """A list of dependencies of task.""" - produces: PyTree[Node] = field(factory=dict) + produces: PyTree[PNode] = field(factory=dict) """A list of products of task.""" markers: list[Mark] = field(factory=list) """A list of markers attached to the task function.""" @@ -166,12 +159,12 @@ def save(self, value: bytes | str) -> None: @define(kw_only=True) -class PythonNode(Node): +class PythonNode(PNode): """The class for a node which is a Python object.""" name: str = "" """Name of the node.""" - value: Any | None = None + value: Any = None """Value of the node.""" hash: bool | Callable[[Any], bool] = False # noqa: A003 """Whether the value should be hashed to determine the state.""" diff --git a/src/_pytask/parameters.py b/src/_pytask/parameters.py index a7b31f9a0..833be73ba 100644 --- a/src/_pytask/parameters.py +++ b/src/_pytask/parameters.py @@ -36,7 +36,7 @@ type=str, multiple=True, help=( - "A pattern to ignore files or directories. Refer to :meth:`pathlib.Path.match` " + "A pattern to ignore files or directories. Refer to 'pathlib.Path.match' " "for more info." ), default=[], diff --git a/src/pytask/__init__.py b/src/pytask/__init__.py index 28ec1e102..4c4f255a5 100644 --- a/src/pytask/__init__.py +++ b/src/pytask/__init__.py @@ -37,6 +37,10 @@ from _pytask.models import CollectionMetadata from _pytask.models import NodeInfo from _pytask.node_protocols import MetaNode +from _pytask.node_protocols import PNode +from _pytask.node_protocols import PPathNode +from _pytask.node_protocols import PTask +from _pytask.node_protocols import PTaskWithPath from _pytask.nodes import PathNode from _pytask.nodes import PythonNode from _pytask.nodes import Task @@ -94,7 +98,11 @@ "NodeNotFoundError", "PathNode", "Persisted", + "PNode", + "PPathNode", "Product", + "PTask", + "PTaskWithPath", "PytaskError", "PythonNode", "ResolvingDependenciesError", diff --git a/tests/test_collect.py b/tests/test_collect.py index f4aa1e9d8..304684a0b 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -430,7 +430,6 @@ def test_setting_name_for_path_node_via_annotation(tmp_path): from pathlib import Path from typing_extensions import Annotated from pytask import Product, PathNode - from typing import Any def task_example( path: Annotated[Path, Product, PathNode(path=Path("out.txt"), name="product")], @@ -485,3 +484,23 @@ def task_example(path: Annotated[Path, Product, node]) -> None: result = runner.invoke(cli, [tmp_path.as_posix()]) assert "ValueError: The value for the parameter 'path'" in result.output + + +@pytest.mark.end_to_end() +def test_relative_path_of_path_node(runner, tmp_path): + source = """ + from pathlib import Path + from typing_extensions import Annotated + from pytask import Product, PathNode + + def task_example( + path: Annotated[Path, Product, PathNode(path=Path("out.txt"), name="product")], + ) -> None: + path.write_text("text") + """ + tmp_path.joinpath("subfolder").mkdir() + tmp_path.joinpath("subfolder", "task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("subfolder", "out.txt").exists() From 0db0b3b94e3e9e4d9ea25269a78cf3cd9a790007 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Sep 2023 19:50:16 +0200 Subject: [PATCH 23/25] Enable relative paths in Jupyter notebooks and for path nodes in general. --- .pre-commit-config.yaml | 1 + docs/source/changes.md | 3 +- src/_pytask/collect.py | 14 ++-- src/_pytask/hookspecs.py | 11 ++- tests/test_collect.py | 6 +- ...functional_interface_w_relative_path.ipynb | 71 +++++++++++++++++++ 6 files changed, 94 insertions(+), 12 deletions(-) create mode 100644 tests/test_jupyter/test_functional_interface_w_relative_path.ipynb diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aec3505f1..ac63e1d22 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,6 +62,7 @@ repos: additional_dependencies: [ attrs>=21.3.0, click, + pluggy, types-setuptools ] pass_filenames: false diff --git a/docs/source/changes.md b/docs/source/changes.md index 0bc5a0621..fcdc005cb 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -28,7 +28,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`408` removes `.value` from `Node` protocol. - {pull}`409` make `.from_annot` an optional feature of nodes. - {pull}`410` allows to pass functions to `PythonNode(hash=...)`. -- {pull}`411` implements a new functional interface. +- {pull}`411` implements a new functional interface and adds experimental support for + defining and running tasks in REPLs or Jupyter notebooks. - {pull}`412` adds protocols for tasks. - {pull}`413` removes scripts to generate `.svg`s. - {pull}`414` allow more ruff rules. diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index a8c857bd3..66ccef313 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -186,7 +186,7 @@ def pytask_collect_file( @hookimpl def pytask_collect_task_protocol( - session: Session, path: Path, name: str, obj: Any + session: Session, path: Path | None, name: str, obj: Any ) -> CollectionReport | None: """Start protocol for collecting a task.""" try: @@ -222,9 +222,11 @@ def pytask_collect_task( """ if (name.startswith("task_") or has_mark(obj, "task")) and callable(obj): - dependencies = parse_dependencies_from_task_function(session, path, name, obj) - - products = parse_products_from_task_function(session, path, name, obj) + path_nodes = Path.cwd() if path is None else path.parent + dependencies = parse_dependencies_from_task_function( + session, path_nodes, name, obj + ) + products = parse_products_from_task_function(session, path_nodes, name, obj) markers = obj.pytask_meta.markers if hasattr(obj, "pytask_meta") else [] @@ -296,7 +298,7 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> PN return node if isinstance(node, PPathNode) and not node.path.is_absolute(): - node.path = path.parent.joinpath(node.path) + node.path = path.joinpath(node.path) # ``normpath`` removes ``../`` from the path which is necessary for the casing # check which will fail since ``.resolves()`` also normalizes a path. @@ -310,7 +312,7 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> PN if isinstance(node, Path): if not node.is_absolute(): - node = path.parent.joinpath(node) + node = path.joinpath(node) # ``normpath`` removes ``../`` from the path which is necessary for the casing # check which will fail since ``.resolves()`` also normalizes a path. diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index 77391aa4f..07f35b172 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -198,7 +198,16 @@ def pytask_collect_task_teardown(session: Session, task: PTask) -> None: def pytask_collect_node( session: Session, path: Path, node_info: NodeInfo ) -> PNode | None: - """Collect a node which is a dependency or a product of a task.""" + """Collect a node which is a dependency or a product of a task. + + Parameters + ---------- + path + The path helps if the path of the node is given relative to the task. The path + either points to the parent directory of the task module or to the current + working directory for tasks defined in the REPL or in Jupyter notebooks. + + """ @hookspec(firstresult=True) diff --git a/tests/test_collect.py b/tests/test_collect.py index 304684a0b..ee1fa07cf 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -181,13 +181,12 @@ def test_pytask_collect_node(session, path, node_info, expected): ) def test_pytask_collect_node_raises_error_if_path_is_not_correctly_cased(tmp_path): session = Session({"check_casing_of_paths": True}, None) - task_path = tmp_path / "task_example.py" real_node = tmp_path / "text.txt" real_node.touch() collected_node = tmp_path / "TeXt.TxT" with pytest.raises(Exception, match="The provided path of"): - pytask_collect_node(session, task_path, NodeInfo("", (), collected_node)) + pytask_collect_node(session, tmp_path, NodeInfo("", (), collected_node)) @pytest.mark.unit() @@ -196,7 +195,6 @@ def test_pytask_collect_node_does_not_raise_error_if_path_is_not_normalized( tmp_path, is_absolute ): session = Session({"check_casing_of_paths": True}, None) - task_path = tmp_path / "task_example.py" real_node = tmp_path / "text.txt" collected_node = Path("..", tmp_path.name, "text.txt") @@ -205,7 +203,7 @@ def test_pytask_collect_node_does_not_raise_error_if_path_is_not_normalized( with warnings.catch_warnings(record=True) as record: result = pytask_collect_node( - session, task_path, NodeInfo("", (), collected_node) + session, tmp_path, NodeInfo("", (), collected_node) ) assert not record diff --git a/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb b/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb new file mode 100644 index 000000000..d942576d8 --- /dev/null +++ b/tests/test_jupyter/test_functional_interface_w_relative_path.ipynb @@ -0,0 +1,71 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "12bc75b1", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "import pytask\n", + "\n", + "from pytask import ExitCode\n", + "from pytask import PathNode\n", + "from pytask import PythonNode\n", + "from typing_extensions import Annotated" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29ac7311", + "metadata": {}, + "outputs": [], + "source": [ + "node_text = PythonNode(name=\"text\", hash=True)\n", + "\n", + "def create_text() -> Annotated[int, node_text]:\n", + " return \"This is the text.\"\n", + "\n", + "node_file = PathNode(name=\"product\", path=Path(\"file.txt\"))\n", + "\n", + "def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]:\n", + " return text" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "738c9418", + "metadata": {}, + "outputs": [], + "source": [ + "session = pytask.build(tasks=[create_file, create_text])\n", + "assert session.exit_code == ExitCode.OK" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 4dd3472d9e91be6bc8443f0892a8368b5df2cd74 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 8 Sep 2023 19:57:50 +0200 Subject: [PATCH 24/25] Move docstrings. --- src/_pytask/collect.py | 7 +++++++ src/_pytask/hookspecs.py | 11 +---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 66ccef313..151ce09b3 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -288,6 +288,13 @@ def pytask_collect_node(session: Session, path: Path, node_info: NodeInfo) -> PN ``trylast=True`` might be necessary if other plugins try to parse strings themselves like a plugin for downloading files which depends on URLs given as strings. + Parameters + ---------- + path + The path helps if the path of the node is given relative to the task. The path + either points to the parent directory of the task module or to the current + working directory for tasks defined in the REPL or in Jupyter notebooks. + """ node = node_info.value diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index 07f35b172..77391aa4f 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -198,16 +198,7 @@ def pytask_collect_task_teardown(session: Session, task: PTask) -> None: def pytask_collect_node( session: Session, path: Path, node_info: NodeInfo ) -> PNode | None: - """Collect a node which is a dependency or a product of a task. - - Parameters - ---------- - path - The path helps if the path of the node is given relative to the task. The path - either points to the parent directory of the task module or to the current - working directory for tasks defined in the REPL or in Jupyter notebooks. - - """ + """Collect a node which is a dependency or a product of a task.""" @hookspec(firstresult=True) From 96d7d981faf0a6f43bbd725aea301719f26bac6f Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 10 Sep 2023 19:31:19 +0200 Subject: [PATCH 25/25] Add test. --- src/_pytask/collect.py | 31 +++++++++++++++++++------------ tests/test_execute.py | 8 ++++++++ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 151ce09b3..95b43e295 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -102,23 +102,30 @@ def _collect_from_tasks(session: Session) -> None: obj=raw_task, ) - if not callable(raw_task): - msg = "Not a function." - raise ValueError(msg) + if callable(raw_task): + if not hasattr(raw_task, "pytask_meta"): + raw_task = task_decorator()(raw_task) # noqa: PLW2901 + + try: + path = get_file(raw_task) + except (TypeError, OSError): + path = None + else: + if path.name == "": + path = None # pragma: no cover - if not hasattr(raw_task, "pytask_meta"): - raw_task = task_decorator()(raw_task) # noqa: PLW2901 + # Detect whether a path is defined in a Jupyter notebook. + if is_jupyter() and "ipykernel" in path.as_posix() and path.suffix == ".py": + path = None # pragma: no cover - path = get_file(raw_task) - if path.name == "": - path = None + name = raw_task.pytask_meta.name - # Detect whether a path is defined in a Jupyter notebook. - if is_jupyter() and "ipykernel" in path.as_posix() and path.suffix == ".py": + # When a task is not a PTask and a callable, set arbitrary values and it will + # pass without errors and not collected. + else: + name = "" path = None - name = raw_task.pytask_meta.name - report = session.hook.pytask_collect_task_protocol( session=session, reports=session.collection_reports, diff --git a/tests/test_execute.py b/tests/test_execute.py index 846184750..7848b62ab 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -8,6 +8,7 @@ import textwrap from pathlib import Path +import pytask import pytest from _pytask.capture import CaptureMethod from _pytask.exceptions import NodeNotFoundError @@ -705,6 +706,7 @@ def task_example() -> Annotated[Dict[str, str], nodes]: assert result.exit_code == ExitCode.OK +@pytest.mark.end_to_end() def test_execute_tasks_and_pass_values_only_by_python_nodes(runner, tmp_path): source = """ from _pytask.nodes import PathNode @@ -763,3 +765,9 @@ def create_file(text: Annotated[int, node_text]) -> Annotated[str, node_file]: ) assert result.returncode == ExitCode.OK assert tmp_path.joinpath("file.txt").read_text() == "This is the text." + + +@pytest.mark.end_to_end() +def test_pass_non_task_to_functional_api_that_are_ignored(): + session = pytask.build(tasks=None) + assert len(session.tasks) == 0