From 43bef2cc83fd5dae94037ae970c8c82000f5e5a5 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 3 Sep 2023 13:13:10 +0200 Subject: [PATCH 01/10] Start writing task protocols. --- src/_pytask/capture.py | 10 +++++----- src/_pytask/clean.py | 4 ++-- src/_pytask/collect.py | 9 +++++---- src/_pytask/collect_command.py | 12 ++++++------ src/_pytask/console.py | 4 ++-- src/_pytask/dag.py | 17 +++++++++------- src/_pytask/dag_utils.py | 4 ++-- src/_pytask/database_utils.py | 4 ++-- src/_pytask/debugging.py | 14 ++++++------- src/_pytask/execute.py | 10 +++++----- src/_pytask/hookspecs.py | 21 ++++++++++---------- src/_pytask/live.py | 10 +++++----- src/_pytask/mark/__init__.py | 6 +++--- src/_pytask/mark_utils.py | 18 ++++++++--------- src/_pytask/node_protocols.py | 36 ++++++++++++++++++++++++++++++++++ src/_pytask/nodes.py | 8 ++++---- src/_pytask/persist.py | 4 ++-- src/_pytask/profile.py | 10 +++++----- src/_pytask/report.py | 8 ++++---- src/_pytask/session.py | 4 ++-- src/_pytask/shared.py | 7 ++++--- src/_pytask/skipping.py | 4 ++-- src/_pytask/warnings.py | 4 ++-- src/_pytask/warnings_utils.py | 9 ++++++--- 24 files changed, 141 insertions(+), 96 deletions(-) diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index 777c76c79..01a9efa8f 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -44,7 +44,7 @@ from _pytask.click import EnumChoice from _pytask.config import hookimpl from _pytask.enums import ShowCapture -from _pytask.nodes import Task +from _pytask.node_protocols import PTask class _CaptureMethod(enum.Enum): @@ -706,7 +706,7 @@ def read(self) -> CaptureResult[str]: # Helper context managers @contextlib.contextmanager - def task_capture(self, when: str, task: Task) -> Generator[None, None, None]: + def task_capture(self, when: str, task: PTask) -> Generator[None, None, None]: """Pipe captured stdout and stderr into report sections.""" self.resume() @@ -722,19 +722,19 @@ def task_capture(self, when: str, task: Task) -> Generator[None, None, None]: # Hooks @hookimpl(hookwrapper=True) - def pytask_execute_task_setup(self, task: Task) -> Generator[None, None, None]: + def pytask_execute_task_setup(self, task: PTask) -> Generator[None, None, None]: """Capture output during setup.""" with self.task_capture("setup", task): yield @hookimpl(hookwrapper=True) - def pytask_execute_task(self, task: Task) -> Generator[None, None, None]: + def pytask_execute_task(self, task: PTask) -> Generator[None, None, None]: """Capture output during execution.""" with self.task_capture("call", task): yield @hookimpl(hookwrapper=True) - def pytask_execute_task_teardown(self, task: Task) -> Generator[None, None, None]: + def pytask_execute_task_teardown(self, task: PTask) -> Generator[None, None, None]: """Capture output during teardown.""" with self.task_capture("teardown", task): yield diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index ecf515626..758b066ce 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -21,7 +21,7 @@ from _pytask.git import get_all_files from _pytask.git import get_root from _pytask.git import is_git_installed -from _pytask.nodes import Task +from _pytask.node_protocols import PTask from _pytask.outcomes import ExitCode from _pytask.path import find_common_ancestor from _pytask.path import relative_to @@ -214,7 +214,7 @@ def _collect_all_paths_known_to_pytask(session: Session) -> set[Path]: return known_paths -def _yield_paths_from_task(task: Task) -> Generator[Path, None, None]: +def _yield_paths_from_task(task: PTask) -> Generator[Path, None, None]: """Yield all paths attached to a task.""" yield task.path for attribute in ("depends_on", "produces"): diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 26db423fd..bf55c0f85 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -22,6 +22,7 @@ from _pytask.mark_utils import has_mark from _pytask.models import NodeInfo from _pytask.node_protocols import Node +from _pytask.node_protocols import PTask from _pytask.nodes import PathNode from _pytask.nodes import PythonNode from _pytask.nodes import Task @@ -270,7 +271,7 @@ def _not_ignored_paths( @hookimpl(trylast=True) -def pytask_collect_modify_tasks(tasks: list[Task]) -> None: +def pytask_collect_modify_tasks(tasks: list[PTask]) -> None: """Given all tasks, assign a short uniquely identifiable name to each task.""" id_to_short_id = _find_shortest_uniquely_identifiable_name_for_tasks(tasks) for task in tasks: @@ -279,7 +280,7 @@ def pytask_collect_modify_tasks(tasks: list[Task]) -> None: def _find_shortest_uniquely_identifiable_name_for_tasks( - tasks: list[Task], + tasks: list[PTask], ) -> dict[str, str]: """Find the shortest uniquely identifiable name for tasks. @@ -313,7 +314,7 @@ def _find_shortest_uniquely_identifiable_name_for_tasks( @hookimpl def pytask_collect_log( - session: Session, reports: list[CollectionReport], tasks: list[Task] + session: Session, reports: list[CollectionReport], tasks: list[PTask] ) -> None: """Log collection.""" session.collection_end = time.time() @@ -334,7 +335,7 @@ def pytask_collect_log( if report.node is None: header = "Error" else: - if isinstance(report.node, Task): + if isinstance(report.node, PTask): short_name = format_task_id( report.node, editor_url_scheme="no_link", short_name=True ) diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 39165beef..9c2073490 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -21,6 +21,7 @@ from _pytask.mark import select_by_keyword from _pytask.mark import select_by_mark from _pytask.node_protocols import PPathNode +from _pytask.node_protocols import PTask from _pytask.outcomes import ExitCode from _pytask.path import find_common_ancestor from _pytask.path import relative_to @@ -33,7 +34,6 @@ if TYPE_CHECKING: from typing import NoReturn - from _pytask.nodes import Task @hookimpl(tryfirst=True) @@ -106,7 +106,7 @@ def collect(**raw_config: Any | None) -> NoReturn: sys.exit(session.exit_code) -def _select_tasks_by_expressions_and_marker(session: Session) -> list[Task]: +def _select_tasks_by_expressions_and_marker(session: Session) -> list[PTask]: """Select tasks by expressions and marker.""" all_tasks = {task.name for task in session.tasks} remaining_by_mark = select_by_mark(session, session.dag) or all_tasks @@ -117,7 +117,7 @@ def _select_tasks_by_expressions_and_marker(session: Session) -> list[Task]: def _find_common_ancestor_of_all_nodes( - tasks: list[Task], paths: list[Path], show_nodes: bool + tasks: list[PTask], paths: list[Path], show_nodes: bool ) -> Path: """Find common ancestor from all nodes and passed paths.""" all_paths = [] @@ -136,14 +136,14 @@ def _find_common_ancestor_of_all_nodes( return common_ancestor -def _organize_tasks(tasks: list[Task]) -> dict[Path, list[Task]]: +def _organize_tasks(tasks: list[PTask]) -> dict[Path, list[PTask]]: """Organize tasks in a dictionary. The dictionary has file names as keys and then a dictionary with task names and below a dictionary with dependencies and targets. """ - dictionary: dict[Path, list[Task]] = {} + dictionary: dict[Path, list[PTask]] = {} for task in tasks: dictionary[task.path] = dictionary.get(task.path, []) dictionary[task.path].append(task) @@ -156,7 +156,7 @@ def _organize_tasks(tasks: list[Task]) -> dict[Path, list[Task]]: def _print_collected_tasks( - dictionary: dict[Path, list[Task]], + dictionary: dict[Path, list[PTask]], show_nodes: bool, editor_url_scheme: str, common_ancestor: Path, diff --git a/src/_pytask/console.py b/src/_pytask/console.py index 461cbb443..2b76f1a56 100644 --- a/src/_pytask/console.py +++ b/src/_pytask/console.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING import rich +from _pytask.node_protocols import PTask from _pytask.path import relative_to as relative_to_ from rich.console import Console from rich.padding import Padding @@ -26,7 +27,6 @@ if TYPE_CHECKING: - from _pytask.nodes import Task from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome @@ -144,7 +144,7 @@ def render_to_string( def format_task_id( - task: Task, + task: PTask, editor_url_scheme: str, short_name: bool = False, relative_to: Path | None = None, diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py index 050aae214..8fc38e4a3 100644 --- a/src/_pytask/dag.py +++ b/src/_pytask/dag.py @@ -1,6 +1,7 @@ """This module contains code related to resolving dependencies.""" from __future__ import annotations +import hashlib import itertools import sys @@ -23,8 +24,8 @@ from _pytask.mark_utils import has_mark from _pytask.node_protocols import MetaNode from _pytask.node_protocols import Node -from _pytask.node_protocols import PPathNode -from _pytask.nodes import Task +from _pytask.node_protocols import PTask +from _pytask.node_protocols import PTaskWithPath from _pytask.path import find_common_ancestor_of_nodes from _pytask.report import DagReport from _pytask.session import Session @@ -32,6 +33,7 @@ from _pytask.shared import reduce_node_name from _pytask.traceback import render_exc_info from _pytask.tree_util import tree_map +from pytask import Task from rich.text import Text from rich.tree import Tree @@ -66,7 +68,7 @@ def pytask_dag(session: Session) -> bool | None: @hookimpl -def pytask_dag_create_dag(tasks: list[Task]) -> nx.DiGraph: +def pytask_dag_create_dag(tasks: list[PTask]) -> nx.DiGraph: """Create the DAG from tasks, dependencies and products.""" dag = nx.DiGraph() @@ -112,7 +114,7 @@ def pytask_dag_validate_dag(dag: nx.DiGraph) -> None: def _have_task_or_neighbors_changed( - session: Session, dag: nx.DiGraph, task: Task + session: Session, dag: nx.DiGraph, task: PTask ) -> bool: """Indicate whether dependencies or products of a task have changed.""" return any( @@ -141,18 +143,19 @@ def pytask_dag_has_node_changed(node: MetaNode, task_name: str) -> bool: if db_state is None: return True - if isinstance(node, (PPathNode, Task)): + # TODO: Could be MetaNode with Path? + if isinstance(node, Task): # If the modification times match, the node has not been changed. if node_state == db_state.modification_time: return False # If the modification time changed, quickly return for non-tasks. - if not isinstance(node, Task): + if not isinstance(node, PTaskWithPath): return True # When modification times changed, we are still comparing the hash of the file # to avoid unnecessary and expensive reexecutions of tasks. - hash_ = node.state(hash=True) + hash_ = hashlib.sha256(node.path.read_bytes()).hexdigest() return hash_ != db_state.hash_ return node_state != db_state.hash_ diff --git a/src/_pytask/dag_utils.py b/src/_pytask/dag_utils.py index e384d10da..28e5e6b48 100644 --- a/src/_pytask/dag_utils.py +++ b/src/_pytask/dag_utils.py @@ -9,7 +9,7 @@ from _pytask.console import format_strings_as_flat_tree from _pytask.console import TASK_ICON from _pytask.mark_utils import has_mark -from _pytask.nodes import Task +from _pytask.node_protocols import PTask from attrs import define from attrs import field @@ -136,7 +136,7 @@ def static_order(self) -> Generator[str, None, None]: self.done(new_task) -def _extract_priorities_from_tasks(tasks: list[Task]) -> dict[str, int]: +def _extract_priorities_from_tasks(tasks: list[PTask]) -> dict[str, int]: """Extract priorities from tasks. Priorities are set via the ``pytask.mark.try_first`` and ``pytask.mark.try_last`` diff --git a/src/_pytask/database_utils.py b/src/_pytask/database_utils.py index 19f5724b7..ffb48a06e 100644 --- a/src/_pytask/database_utils.py +++ b/src/_pytask/database_utils.py @@ -5,7 +5,7 @@ from _pytask.dag_utils import node_and_neighbors from _pytask.node_protocols import PPathNode -from _pytask.nodes import Task +from _pytask.node_protocols import PTaskWithPath from _pytask.session import Session from sqlalchemy import Column from sqlalchemy import create_engine @@ -77,7 +77,7 @@ def update_states_in_database(session: Session, task_name: str) -> None: for name in node_and_neighbors(session.dag, task_name): node = session.dag.nodes[name].get("task") or session.dag.nodes[name]["node"] - if isinstance(node, Task): + if isinstance(node, PTaskWithPath): modification_time = node.state() hash_ = hashlib.sha256(node.path.read_bytes()).hexdigest() elif isinstance(node, PPathNode): diff --git a/src/_pytask/debugging.py b/src/_pytask/debugging.py index 78f2eb673..13ceadf7b 100644 --- a/src/_pytask/debugging.py +++ b/src/_pytask/debugging.py @@ -15,7 +15,7 @@ import pluggy from _pytask.config import hookimpl from _pytask.console import console -from _pytask.nodes import Task +from _pytask.node_protocols import PTask from _pytask.outcomes import Exit from _pytask.session import Session from _pytask.traceback import remove_internal_traceback_frames_from_exc_info @@ -310,15 +310,15 @@ class PdbDebugger: @staticmethod @hookimpl(hookwrapper=True) def pytask_execute_task( - session: Session, task: Task + session: Session, task: PTask ) -> Generator[None, None, None]: """Execute a task by wrapping the function with post-mortem debugger.""" - if isinstance(task, Task): + if isinstance(task, PTask): wrap_function_for_post_mortem_debugging(session, task) yield -def wrap_function_for_post_mortem_debugging(session: Session, task: Task) -> None: +def wrap_function_for_post_mortem_debugging(session: Session, task: PTask) -> None: """Wrap the function for post-mortem debugging.""" task_function = task.function @@ -370,15 +370,15 @@ class PdbTrace: @staticmethod @hookimpl(hookwrapper=True) def pytask_execute_task( - session: Session, task: Task + session: Session, task: PTask ) -> Generator[None, None, None]: """Wrapping the task function with a tracer.""" - if isinstance(task, Task): + if isinstance(task, PTask): wrap_function_for_tracing(session, task) yield -def wrap_function_for_tracing(session: Session, task: Task) -> None: +def wrap_function_for_tracing(session: Session, task: PTask) -> None: """Wrap the task function for tracing.""" _pdb = PytaskPDB._init_pdb("runcall") task_function = task.function diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index 6603d33e2..c2ab82abc 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -22,7 +22,7 @@ from _pytask.mark import Mark from _pytask.mark_utils import has_mark from _pytask.node_protocols import PPathNode -from _pytask.nodes import Task +from _pytask.node_protocols import PTask from _pytask.outcomes import count_outcomes from _pytask.outcomes import Exit from _pytask.outcomes import TaskOutcome @@ -90,7 +90,7 @@ def pytask_execute_build(session: Session) -> bool: @hookimpl -def pytask_execute_task_protocol(session: Session, task: Task) -> ExecutionReport: +def pytask_execute_task_protocol(session: Session, task: PTask) -> ExecutionReport: """Follow the protocol to execute each task.""" session.hook.pytask_execute_task_log_start(session=session, task=task) try: @@ -112,7 +112,7 @@ def pytask_execute_task_protocol(session: Session, task: Task) -> ExecutionRepor @hookimpl(trylast=True) -def pytask_execute_task_setup(session: Session, task: Task) -> None: +def pytask_execute_task_setup(session: Session, task: PTask) -> None: """Set up the execution of a task. 1. Check whether all dependencies of a task are available. @@ -138,7 +138,7 @@ def pytask_execute_task_setup(session: Session, task: Task) -> None: @hookimpl(trylast=True) -def pytask_execute_task(session: Session, task: Task) -> bool: +def pytask_execute_task(session: Session, task: PTask) -> bool: """Execute task.""" if session.config["dry_run"]: raise WouldBeExecuted @@ -158,7 +158,7 @@ def pytask_execute_task(session: Session, task: Task) -> bool: @hookimpl -def pytask_execute_task_teardown(session: Session, task: Task) -> None: +def pytask_execute_task_teardown(session: Session, task: PTask) -> None: """Check if :class:`_pytask.nodes.PathNode` are produced by a task.""" missing_nodes = [] for product in session.dag.successors(task.name): diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index dbd565154..840939263 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -16,6 +16,7 @@ from _pytask.models import NodeInfo from _pytask.node_protocols import MetaNode from _pytask.node_protocols import Node +from _pytask.node_protocols import PTask if TYPE_CHECKING: @@ -128,7 +129,7 @@ def pytask_ignore_collect(path: pathlib.Path, config: dict[str, Any]) -> bool: @hookspec -def pytask_collect_modify_tasks(session: Session, tasks: list[Task]) -> None: +def pytask_collect_modify_tasks(session: Session, tasks: list[PTask]) -> None: """Modify tasks after they have been collected. This hook can be used to deselect tasks when they match a certain keyword or mark. @@ -186,7 +187,7 @@ def pytask_collect_task( @hookspec -def pytask_collect_task_teardown(session: Session, task: Task) -> None: +def pytask_collect_task_teardown(session: Session, task: PTask) -> None: """Perform tear-down operations when a task was collected. Use this hook specification to, for example, perform checks on the collected task. @@ -203,7 +204,7 @@ def pytask_collect_node( @hookspec(firstresult=True) def pytask_collect_log( - session: Session, reports: list[CollectionReport], tasks: list[Task] + session: Session, reports: list[CollectionReport], tasks: list[PTask] ) -> None: """Log errors occurring during the collection. @@ -226,7 +227,7 @@ def pytask_dag(session: Session) -> None: @hookspec(firstresult=True) -def pytask_dag_create_dag(session: Session, tasks: list[Task]) -> nx.DiGraph: +def pytask_dag_create_dag(session: Session, tasks: list[PTask]) -> nx.DiGraph: """Create the DAG. This hook creates the DAG from tasks, dependencies and products. The DAG can be used @@ -324,7 +325,7 @@ def pytask_execute_build(session: Session) -> Any: @hookspec(firstresult=True) -def pytask_execute_task_protocol(session: Session, task: Task) -> ExecutionReport: +def pytask_execute_task_protocol(session: Session, task: PTask) -> ExecutionReport: """Run the protocol for executing a test. This hook runs all stages of the execution process, setup, execution, and teardown @@ -336,7 +337,7 @@ def pytask_execute_task_protocol(session: Session, task: Task) -> ExecutionRepor @hookspec(firstresult=True) -def pytask_execute_task_log_start(session: Session, task: Task) -> None: +def pytask_execute_task_log_start(session: Session, task: PTask) -> None: """Start logging of task execution. This hook can be used to provide more verbose output during the execution. @@ -345,7 +346,7 @@ def pytask_execute_task_log_start(session: Session, task: Task) -> None: @hookspec -def pytask_execute_task_setup(session: Session, task: Task) -> None: +def pytask_execute_task_setup(session: Session, task: PTask) -> None: """Set up the task execution. This hook is called before the task is executed and can provide an entry-point to @@ -356,12 +357,12 @@ def pytask_execute_task_setup(session: Session, task: Task) -> None: @hookspec(firstresult=True) -def pytask_execute_task(session: Session, task: Task) -> Any: +def pytask_execute_task(session: Session, task: PTask) -> Any: """Execute a task.""" @hookspec -def pytask_execute_task_teardown(session: Session, task: Task) -> None: +def pytask_execute_task_teardown(session: Session, task: PTask) -> None: """Tear down task execution. This hook is executed after the task has been executed. It allows to perform clean- @@ -417,7 +418,7 @@ def pytask_log_session_footer( @hookspec def pytask_profile_add_info_on_task( - session: Session, tasks: list[Task], profile: dict[str, dict[Any, Any]] + session: Session, tasks: list[PTask], profile: dict[str, dict[Any, Any]] ) -> None: """Add information on task for profile. diff --git a/src/_pytask/live.py b/src/_pytask/live.py index af928a6f7..98f5571e9 100644 --- a/src/_pytask/live.py +++ b/src/_pytask/live.py @@ -9,7 +9,7 @@ from _pytask.config import hookimpl from _pytask.console import console from _pytask.console import format_task_id -from _pytask.nodes import Task +from _pytask.node_protocols import PTask from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome from _pytask.report import CollectionReport @@ -128,7 +128,7 @@ def is_started(self) -> None: class _ReportEntry(NamedTuple): name: str outcome: TaskOutcome - task: Task + task: PTask @define(eq=False, kw_only=True) @@ -142,7 +142,7 @@ class LiveExecution: sort_final_table: bool = False n_tasks: int | str = "x" _reports: list[_ReportEntry] = field(factory=list) - _running_tasks: dict[str, Task] = field(factory=dict) + _running_tasks: dict[str, PTask] = field(factory=dict) @hookimpl(hookwrapper=True) def pytask_execute_build(self) -> Generator[None, None, None]: @@ -157,7 +157,7 @@ def pytask_execute_build(self) -> Generator[None, None, None]: console.print(table) @hookimpl(tryfirst=True) - def pytask_execute_task_log_start(self, task: Task) -> bool: + def pytask_execute_task_log_start(self, task: PTask) -> bool: """Mark a new task as running.""" self.update_running_tasks(task) return True @@ -256,7 +256,7 @@ def _update_table( ) self.live_manager.update(table) - def update_running_tasks(self, new_running_task: Task) -> None: + def update_running_tasks(self, new_running_task: PTask) -> None: """Add a new running task.""" self._running_tasks[new_running_task.name] = new_running_task self._update_table() diff --git a/src/_pytask/mark/__init__.py b/src/_pytask/mark/__init__.py index 83a603291..3839dbab7 100644 --- a/src/_pytask/mark/__init__.py +++ b/src/_pytask/mark/__init__.py @@ -19,7 +19,7 @@ from _pytask.mark.structures import MARK_GEN from _pytask.mark.structures import MarkDecorator from _pytask.mark.structures import MarkGenerator -from _pytask.nodes import Task +from _pytask.node_protocols import PTask from _pytask.outcomes import ExitCode from _pytask.pluginmanager import get_plugin_manager from _pytask.session import Session @@ -134,7 +134,7 @@ class KeywordMatcher: _names: AbstractSet[str] @classmethod - def from_task(cls, task: Task) -> KeywordMatcher: + def from_task(cls, task: PTask) -> KeywordMatcher: mapped_names = {task.name} # Add the names attached to the current function through direct assignment. @@ -186,7 +186,7 @@ class MarkMatcher: own_mark_names: set[str] @classmethod - def from_task(cls, task: Task) -> MarkMatcher: + def from_task(cls, task: PTask) -> MarkMatcher: mark_names = {mark.name for mark in task.markers} return cls(mark_names) diff --git a/src/_pytask/mark_utils.py b/src/_pytask/mark_utils.py index cdbd9d61e..348493c53 100644 --- a/src/_pytask/mark_utils.py +++ b/src/_pytask/mark_utils.py @@ -9,16 +9,16 @@ from typing import TYPE_CHECKING from _pytask.models import CollectionMetadata -from _pytask.nodes import Task +from _pytask.node_protocols import PTask if TYPE_CHECKING: from _pytask.mark import Mark -def get_all_marks(obj_or_task: Any | Task) -> list[Mark]: +def get_all_marks(obj_or_task: Any | PTask) -> list[Mark]: """Get all marks from a callable or task.""" - if isinstance(obj_or_task, Task): + if isinstance(obj_or_task, PTask): marks = obj_or_task.markers else: obj = obj_or_task @@ -26,9 +26,9 @@ def get_all_marks(obj_or_task: Any | Task) -> list[Mark]: return marks -def set_marks(obj_or_task: Any | Task, marks: list[Mark]) -> Any | Task: +def set_marks(obj_or_task: Any | PTask, marks: list[Mark]) -> Any | PTask: """Set marks on a callable or task.""" - if isinstance(obj_or_task, Task): + if isinstance(obj_or_task, PTask): obj_or_task.markers = marks elif hasattr(obj_or_task, "pytask_meta"): obj_or_task.pytask_meta.markers = marks @@ -37,21 +37,21 @@ def set_marks(obj_or_task: Any | Task, marks: list[Mark]) -> Any | Task: return obj_or_task -def get_marks(obj_or_task: Any | Task, marker_name: str) -> list[Mark]: +def get_marks(obj_or_task: Any | PTask, marker_name: str) -> list[Mark]: """Get marks from callable or task.""" marks = get_all_marks(obj_or_task) return [mark for mark in marks if mark.name == marker_name] -def has_mark(obj_or_task: Any | Task, marker_name: str) -> bool: +def has_mark(obj_or_task: Any | PTask, marker_name: str) -> bool: """Test if callable or task has a certain mark.""" marks = get_all_marks(obj_or_task) return any(mark.name == marker_name for mark in marks) def remove_marks( - obj_or_task: Any | Task, marker_name: str -) -> tuple[Any | Task, list[Mark]]: + obj_or_task: Any | PTask, marker_name: str +) -> tuple[Any | PTask, list[Mark]]: """Remove marks from callable or task.""" marks = get_all_marks(obj_or_task) selected = [mark for mark in marks if mark.name == marker_name] diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py index b8660a874..70165da13 100644 --- a/src/_pytask/node_protocols.py +++ b/src/_pytask/node_protocols.py @@ -3,9 +3,13 @@ from abc import abstractmethod from pathlib import Path from typing import Any +from typing import Callable from typing import Protocol from typing import runtime_checkable +from _pytask.mark import Mark +from _pytask.tree_util import PyTree + @runtime_checkable class MetaNode(Protocol): @@ -47,3 +51,35 @@ class PPathNode(Node, Protocol): """ path: Path + + +@runtime_checkable +class PTask(MetaNode, Protocol): + """Protocol for nodes.""" + + name: str + depends_on: PyTree[Node] + produces: PyTree[Node] + markers: list[Mark] + _report_sections: list[tuple[str, str, str]] + attributes: dict[Any, Any] + function: Callable[..., Any] + + def execute(self, **kwargs: Any) -> Any: + """Return the value of the node that will be injected into the task.""" + ... + + def add_report_section(self, when: str, key: str, content: str) -> None: + """Add sections which will be displayed in report like stdout and stderr.""" + ... + + +@runtime_checkable +class PTaskWithPath(PTask, Protocol): + """Tasks with paths. + + Tasks with paths receive special handling when it comes to printing their names. + + """ + + path: Path diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index c653304aa..fff2e2c7c 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -67,11 +67,9 @@ def __attrs_post_init__(self: Task) -> None: if self.short_name is None: self.short_name = self.name - def state(self, hash: bool = False) -> str | None: # noqa: A002 + def state(self) -> str | None: """Return the state of the node.""" - if hash and self.path.exists(): - return hashlib.sha256(self.path.read_bytes()).hexdigest() - if not hash and self.path.exists(): + if self.path.exists(): return str(self.path.stat().st_mtime) return None @@ -79,6 +77,7 @@ def execute(self, **kwargs: Any) -> None: """Execute the task.""" out = self.function(**kwargs) + # TODO: Move to pytask_execute_task. if "return" in self.produces: structure_out = tree_structure(out) structure_return = tree_structure(self.produces["return"]) @@ -97,6 +96,7 @@ def execute(self, **kwargs: Any) -> None: def add_report_section(self, when: str, key: str, content: str) -> None: """Add sections which will be displayed in report like stdout or stderr.""" + # TODO: Remove method. if content: self._report_sections.append((when, key, content)) diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py index 2673e1320..f76faa27f 100644 --- a/src/_pytask/persist.py +++ b/src/_pytask/persist.py @@ -8,13 +8,13 @@ from _pytask.dag_utils import node_and_neighbors from _pytask.database_utils import update_states_in_database from _pytask.mark_utils import has_mark +from _pytask.node_protocols import PTask from _pytask.outcomes import Persisted from _pytask.outcomes import TaskOutcome if TYPE_CHECKING: from _pytask.session import Session - from _pytask.nodes import Task from _pytask.report import ExecutionReport @@ -31,7 +31,7 @@ def pytask_parse_config(config: dict[str, Any]) -> None: @hookimpl -def pytask_execute_task_setup(session: Session, task: Task) -> None: +def pytask_execute_task_setup(session: Session, task: PTask) -> None: """Exit persisting tasks early. The decorator needs to be set and all nodes need to exist. diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 5c8a33e57..fade69c35 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -24,7 +24,7 @@ from _pytask.exceptions import CollectionError from _pytask.exceptions import ConfigurationError from _pytask.node_protocols import PPathNode -from _pytask.nodes import Task +from _pytask.node_protocols import PTask from _pytask.outcomes import ExitCode from _pytask.outcomes import TaskOutcome from _pytask.pluginmanager import get_plugin_manager @@ -72,7 +72,7 @@ def pytask_post_parse(config: dict[str, Any]) -> None: @hookimpl(hookwrapper=True) -def pytask_execute_task(task: Task) -> Generator[None, None, None]: +def pytask_execute_task(task: PTask) -> Generator[None, None, None]: """Attach the duration of the execution to the task.""" start = time.time() yield @@ -165,7 +165,7 @@ def profile(**raw_config: Any) -> NoReturn: def _print_profile_table( - profile: dict[str, dict[str, Any]], tasks: list[Task], config: dict[str, Any] + profile: dict[str, dict[str, Any]], tasks: list[PTask], config: dict[str, Any] ) -> None: """Print the profile table.""" name_to_task = {task.name: task for task in tasks} @@ -197,7 +197,7 @@ class DurationNameSpace: @staticmethod @hookimpl def pytask_profile_add_info_on_task( - tasks: list[Task], profile: dict[str, dict[str, Any]] + tasks: list[PTask], profile: dict[str, dict[str, Any]] ) -> None: """Add the runtime for tasks to the profile.""" runtimes = _collect_runtimes([task.name for task in tasks]) @@ -219,7 +219,7 @@ class FileSizeNameSpace: @staticmethod @hookimpl def pytask_profile_add_info_on_task( - session: Session, tasks: list[Task], profile: dict[str, dict[str, Any]] + session: Session, tasks: list[PTask], profile: dict[str, dict[str, Any]] ) -> None: """Add the total file size of all products for a task.""" for task in tasks: diff --git a/src/_pytask/report.py b/src/_pytask/report.py index 7df151fc3..dd9882226 100644 --- a/src/_pytask/report.py +++ b/src/_pytask/report.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING from typing import Union +from _pytask.node_protocols import PTask from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome from _pytask.traceback import remove_internal_traceback_frames_from_exc_info @@ -16,7 +17,6 @@ if TYPE_CHECKING: from _pytask.node_protocols import MetaNode - from _pytask.nodes import Task ExceptionInfo = Tuple[Type[BaseException], BaseException, Union[TracebackType, None]] @@ -57,20 +57,20 @@ def from_exception(cls, exc_info: ExceptionInfo) -> DagReport: class ExecutionReport: """A report for an executed task.""" - task: Task + task: PTask outcome: TaskOutcome exc_info: ExceptionInfo | None = None sections: list[tuple[str, str, str]] = field(factory=list) @classmethod def from_task_and_exception( - cls, task: Task, exc_info: ExceptionInfo + cls, task: PTask, exc_info: ExceptionInfo ) -> ExecutionReport: """Create a report from a task and an exception.""" exc_info = remove_internal_traceback_frames_from_exc_info(exc_info) return cls(task, TaskOutcome.FAIL, exc_info, task._report_sections) @classmethod - def from_task(cls, task: Task) -> ExecutionReport: + def from_task(cls, task: PTask) -> ExecutionReport: """Create a report from a task.""" return cls(task, TaskOutcome.SUCCESS, None, task._report_sections) diff --git a/src/_pytask/session.py b/src/_pytask/session.py index f486a19a2..9d1adf780 100644 --- a/src/_pytask/session.py +++ b/src/_pytask/session.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING import networkx as nx +from _pytask.node_protocols import PTask from _pytask.outcomes import ExitCode from _pytask.warnings_utils import WarningReport from attrs import define @@ -21,7 +22,6 @@ from _pytask.report import CollectionReport from _pytask.report import ExecutionReport from _ptytask.report import DagReport - from _pytask.nodes import Task @define @@ -38,7 +38,7 @@ class Session: The reports capture errors which happened while collecting tasks. """ - tasks: list[Task] = field(factory=list) + tasks: list[PTask] = field(factory=list) """List of collected tasks.""" dag: nx.DiGraph | None = None resolving_dependencies_report: DagReport | None = None diff --git a/src/_pytask/shared.py b/src/_pytask/shared.py index 465f15b35..4ca47ce79 100644 --- a/src/_pytask/shared.py +++ b/src/_pytask/shared.py @@ -12,7 +12,8 @@ from _pytask.console import format_task_id from _pytask.node_protocols import MetaNode from _pytask.node_protocols import PPathNode -from _pytask.nodes import Task +from _pytask.node_protocols import PTask +from _pytask.node_protocols import PTaskWithPath from _pytask.path import find_closest_ancestor from _pytask.path import find_common_ancestor from _pytask.path import relative_to @@ -69,7 +70,7 @@ def reduce_node_name(node: MetaNode, paths: Sequence[str | Path]) -> str: path from one path in ``session.config["paths"]`` to the node. """ - if isinstance(node, (PPathNode, Task)): + if isinstance(node, (PPathNode, PTaskWithPath)): ancestor = find_closest_ancestor(node.path, paths) if ancestor is None: try: @@ -90,7 +91,7 @@ def reduce_names_of_multiple_nodes( for name in names: node = dag.nodes[name].get("node") or dag.nodes[name].get("task") - if isinstance(node, Task): + if isinstance(node, PTask): short_name = format_task_id( node, editor_url_scheme="no_link", short_name=True ) diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py index 6dcea011f..30892a2c1 100644 --- a/src/_pytask/skipping.py +++ b/src/_pytask/skipping.py @@ -9,6 +9,7 @@ from _pytask.mark import Mark from _pytask.mark_utils import get_marks from _pytask.mark_utils import has_mark +from _pytask.node_protocols import PTask from _pytask.outcomes import Skipped from _pytask.outcomes import SkippedAncestorFailed from _pytask.outcomes import SkippedUnchanged @@ -18,7 +19,6 @@ if TYPE_CHECKING: from _pytask.session import Session - from _pytask.nodes import Task from _pytask.report import ExecutionReport @@ -47,7 +47,7 @@ def pytask_parse_config(config: dict[str, Any]) -> None: @hookimpl -def pytask_execute_task_setup(session: Session, task: Task) -> None: +def pytask_execute_task_setup(session: Session, task: PTask) -> None: """Take a short-cut for skipped tasks during setup with an exception.""" is_unchanged = has_mark(task, "skip_unchanged") and not has_mark( task, "would_be_executed" diff --git a/src/_pytask/warnings.py b/src/_pytask/warnings.py index 5e3c92cfe..81728186d 100644 --- a/src/_pytask/warnings.py +++ b/src/_pytask/warnings.py @@ -8,7 +8,7 @@ import click from _pytask.config import hookimpl from _pytask.console import console -from _pytask.nodes import Task +from _pytask.node_protocols import PTask from _pytask.session import Session from _pytask.warnings_utils import catch_warnings_for_item from _pytask.warnings_utils import parse_filterwarnings @@ -63,7 +63,7 @@ def pytask_collect(session: Session) -> Generator[None, None, None]: @staticmethod @hookimpl(hookwrapper=True) def pytask_execute_task( - session: Session, task: Task + session: Session, task: PTask ) -> Generator[None, None, None]: """Catch warnings while executing a task.""" with catch_warnings_for_item(session=session, task=task): diff --git a/src/_pytask/warnings_utils.py b/src/_pytask/warnings_utils.py index 45a9e2862..739a49ab1 100644 --- a/src/_pytask/warnings_utils.py +++ b/src/_pytask/warnings_utils.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING from _pytask.mark_utils import get_marks -from _pytask.nodes import Task +from _pytask.node_protocols import PTask from _pytask.outcomes import Exit @@ -150,7 +150,7 @@ def parse_filterwarnings(x: str | list[str] | None) -> list[str]: @contextmanager def catch_warnings_for_item( session: Session, - task: Task | None = None, + task: PTask | None = None, when: str | None = None, ) -> Generator[None, None, None]: """Context manager that catches warnings generated in the contained execution block. @@ -173,7 +173,10 @@ def catch_warnings_for_item( yield - id_ = task.short_name if task is not None else when + if task is not None: + id_ = task.short_name if hasattr(task, "short_name") else task.name + else: + id_ = when for warning_message in log: fs_location = warning_message.filename, warning_message.lineno From e4fe479cbcae849567103ab5dead08662a544d99 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 3 Sep 2023 19:18:53 +0200 Subject: [PATCH 02/10] temp commit. --- src/_pytask/dag.py | 3 ++- src/_pytask/node_protocols.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py index 8fc38e4a3..753725842 100644 --- a/src/_pytask/dag.py +++ b/src/_pytask/dag.py @@ -24,6 +24,7 @@ from _pytask.mark_utils import has_mark from _pytask.node_protocols import MetaNode from _pytask.node_protocols import Node +from _pytask.node_protocols import PPathNode from _pytask.node_protocols import PTask from _pytask.node_protocols import PTaskWithPath from _pytask.path import find_common_ancestor_of_nodes @@ -144,7 +145,7 @@ def pytask_dag_has_node_changed(node: MetaNode, task_name: str) -> bool: return True # TODO: Could be MetaNode with Path? - if isinstance(node, Task): + if isinstance(node, (PPathNode, Task)): # If the modification times match, the node has not been changed. if node_state == db_state.modification_time: return False diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py index 70165da13..ee992b8ae 100644 --- a/src/_pytask/node_protocols.py +++ b/src/_pytask/node_protocols.py @@ -6,10 +6,13 @@ from typing import Callable from typing import Protocol from typing import runtime_checkable +from typing import TYPE_CHECKING -from _pytask.mark import Mark from _pytask.tree_util import PyTree +if TYPE_CHECKING: + from _pytask.mark import Mark + @runtime_checkable class MetaNode(Protocol): From 517da097fe34b8318dd17ad3450b89165be9105a Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 3 Sep 2023 21:03:15 +0200 Subject: [PATCH 03/10] Simplify format_task_id. --- src/_pytask/collect.py | 2 +- src/_pytask/collect_command.py | 5 +---- src/_pytask/console.py | 29 ++++++++++----------------- src/_pytask/execute.py | 4 +--- src/_pytask/live.py | 10 ++-------- src/_pytask/profile.py | 1 - src/_pytask/shared.py | 4 +--- tests/test_console.py | 36 +++------------------------------- 8 files changed, 19 insertions(+), 72 deletions(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index bf55c0f85..895b217e1 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -337,7 +337,7 @@ def pytask_collect_log( else: if isinstance(report.node, PTask): short_name = format_task_id( - report.node, editor_url_scheme="no_link", short_name=True + report.node, editor_url_scheme="no_link" ) else: short_name = reduce_node_name(report.node, session.config["paths"]) diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 9c2073490..341daf52e 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -192,10 +192,7 @@ def _print_collected_tasks( for task in tasks: reduced_task_name = format_task_id( - task, - editor_url_scheme=editor_url_scheme, - short_name=True, - relative_to=common_ancestor, + task, editor_url_scheme=editor_url_scheme ) task_branch = module_branch.add( Text.assemble(TASK_ICON, ""), diff --git a/src/_pytask/console.py b/src/_pytask/console.py index 2b76f1a56..1b5b4f0c6 100644 --- a/src/_pytask/console.py +++ b/src/_pytask/console.py @@ -14,7 +14,7 @@ import rich from _pytask.node_protocols import PTask -from _pytask.path import relative_to as relative_to_ +from _pytask.nodes import Task from rich.console import Console from rich.padding import Padding from rich.panel import Panel @@ -143,29 +143,20 @@ def render_to_string( return rendered -def format_task_id( - task: PTask, - editor_url_scheme: str, - short_name: bool = False, - relative_to: Path | None = None, -) -> Text: +def format_task_id(task: PTask, editor_url_scheme: str) -> Text: """Format a task id.""" - if short_name: - path, task_name = task.short_name.split("::") - elif relative_to: - path = relative_to_(task.path, relative_to).as_posix() - task_name = task.base_name - else: - path, task_name = task.name.split("::") - if task.function is None: url_style = Style() else: url_style = create_url_style_for_task(task.function, editor_url_scheme) - task_id = Text.assemble( - Text(path + "::", style="dim"), Text(task_name, style=url_style) - ) + if isinstance(task, Task): + path, task_name = task.short_name.split("::") + task_id = Text.assemble( + Text(path + "::", style="dim"), Text(task_name, style=url_style) + ) + else: + task_id = Text(task.name, style=url_style) return task_id @@ -173,7 +164,7 @@ def format_strings_as_flat_tree(strings: Iterable[str], title: str, icon: str) - """Format list of strings as flat tree.""" tree = Tree(title) for name in strings: - tree.add(icon + name) + tree.add(Text.assemble(icon, name)) text = render_to_string(tree, console=console) return text diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index c2ab82abc..bc504e170 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -288,9 +288,7 @@ def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> def _print_errored_task_report(session: Session, report: ExecutionReport) -> None: """Print the traceback and the exception of an errored report.""" task_name = format_task_id( - task=report.task, - editor_url_scheme=session.config["editor_url_scheme"], - short_name=True, + task=report.task, editor_url_scheme=session.config["editor_url_scheme"] ) text = Text.assemble("Task ", task_name, " failed", style="failed") console.rule(text, style=report.outcome.style) diff --git a/src/_pytask/live.py b/src/_pytask/live.py index 98f5571e9..6aeaa8e84 100644 --- a/src/_pytask/live.py +++ b/src/_pytask/live.py @@ -223,18 +223,12 @@ def _generate_table( table.add_column("Outcome") for report in relevant_reports: table.add_row( - format_task_id( - report.task, - editor_url_scheme=self.editor_url_scheme, - short_name=True, - ), + format_task_id(report.task, editor_url_scheme=self.editor_url_scheme), Text(report.outcome.symbol, style=report.outcome.style), ) for task in self._running_tasks.values(): table.add_row( - format_task_id( - task, editor_url_scheme=self.editor_url_scheme, short_name=True - ), + format_task_id(task, editor_url_scheme=self.editor_url_scheme), "running", ) diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index fade69c35..9327a5cfc 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -181,7 +181,6 @@ def _print_profile_table( task_id = format_task_id( task=name_to_task[task_name], editor_url_scheme=config["editor_url_scheme"], - short_name=True, ) infos = [str(i) for i in info.values()] table.add_row(task_id, *infos) diff --git a/src/_pytask/shared.py b/src/_pytask/shared.py index 4ca47ce79..0ffaf722a 100644 --- a/src/_pytask/shared.py +++ b/src/_pytask/shared.py @@ -92,9 +92,7 @@ def reduce_names_of_multiple_nodes( node = dag.nodes[name].get("node") or dag.nodes[name].get("task") if isinstance(node, PTask): - short_name = format_task_id( - node, editor_url_scheme="no_link", short_name=True - ) + short_name = format_task_id(node, editor_url_scheme="no_link") elif isinstance(node, MetaNode): short_name = reduce_node_name(node, paths) else: diff --git a/tests/test_console.py b/tests/test_console.py index bd0b5eca6..ea8f86714 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -114,8 +114,6 @@ def test_render_to_string(color_system, text, strip_styles, expected): "base_name", "short_name", "editor_url_scheme", - "use_short_name", - "relative_to", "expected", ), [ @@ -123,46 +121,18 @@ def test_render_to_string(color_system, text, strip_styles, expected): "task_a", None, "no_link", - False, - None, Text( _THIS_FILE.as_posix() + "::task_a", spans=[Span(0, len(_THIS_FILE.as_posix()) + 2, "dim")], ), id="format full id", - ), - pytest.param( - "task_a", - "test_console.py::task_a", - "no_link", - True, - None, - Text( - "test_console.py::task_a", - spans=[Span(0, 17, "dim")], - ), - id="format short id", - ), - pytest.param( - "task_a", - None, - "no_link", - False, - _THIS_FILE.parent, - Text( - "tests/test_console.py::task_a", - spans=[Span(0, 23, "dim")], - ), - id="format relative to id", - ), + ) ], ) -def test_format_task_id( # noqa: PLR0913 +def test_format_task_id( base_name, short_name, editor_url_scheme, - use_short_name, - relative_to, expected, ): path = _THIS_FILE @@ -171,7 +141,7 @@ def test_format_task_id( # noqa: PLR0913 if short_name is not None: task.short_name = short_name - result = format_task_id(task, editor_url_scheme, use_short_name, relative_to) + result = format_task_id(task, editor_url_scheme) assert result == expected From dbec37488695b045d31d3f8cde71d37e104d63d3 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 3 Sep 2023 21:09:17 +0200 Subject: [PATCH 04/10] Use format_task_id in dag_utils. --- src/_pytask/dag_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/_pytask/dag_utils.py b/src/_pytask/dag_utils.py index 28e5e6b48..aea3385bd 100644 --- a/src/_pytask/dag_utils.py +++ b/src/_pytask/dag_utils.py @@ -7,6 +7,7 @@ import networkx as nx from _pytask.console import format_strings_as_flat_tree +from _pytask.console import format_task_id from _pytask.console import TASK_ICON from _pytask.mark_utils import has_mark from _pytask.node_protocols import PTask @@ -159,9 +160,11 @@ def _extract_priorities_from_tasks(tasks: list[PTask]) -> dict[str, int]: if tasks_w_mixed_priorities: name_to_task = {task.name: task for task in tasks} - reduced_names = [ - name_to_task[name].short_name for name in tasks_w_mixed_priorities - ] + reduced_names = [] + for name in tasks_w_mixed_priorities: + reduced_name = format_task_id(name_to_task[name], "no_link") + reduced_names.append(reduced_name.plain) + text = format_strings_as_flat_tree( reduced_names, "Tasks with mixed priorities", TASK_ICON ) From 7094816898f923768c148983d200773db79e030e Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 3 Sep 2023 22:35:05 +0200 Subject: [PATCH 05/10] Some fixes. --- src/_pytask/clean.py | 3 ++- src/_pytask/collect.py | 10 +++++----- src/_pytask/collect_command.py | 4 ++-- src/_pytask/console.py | 9 +++++---- src/_pytask/dag_utils.py | 4 ++-- src/_pytask/execute.py | 4 ++-- src/_pytask/live.py | 8 ++++---- src/_pytask/nodes.py | 6 +++--- src/_pytask/profile.py | 4 ++-- src/_pytask/shared.py | 4 ++-- src/_pytask/warnings_utils.py | 5 +---- tests/test_console.py | 6 +++--- tests/test_live.py | 14 +++++++------- 13 files changed, 40 insertions(+), 41 deletions(-) diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index 758b066ce..295a291a2 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -21,6 +21,7 @@ from _pytask.git import get_all_files from _pytask.git import get_root from _pytask.git import is_git_installed +from _pytask.node_protocols import PPathNode from _pytask.node_protocols import PTask from _pytask.outcomes import ExitCode from _pytask.path import find_common_ancestor @@ -219,7 +220,7 @@ def _yield_paths_from_task(task: PTask) -> Generator[Path, None, None]: yield task.path for attribute in ("depends_on", "produces"): for node in tree_leaves(getattr(task, attribute)): - if hasattr(node, "path") and isinstance(node.path, Path): + if isinstance(node, PPathNode): yield node.path diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 895b217e1..1e5f1eba1 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -17,7 +17,7 @@ from _pytask.config import IS_FILE_SYSTEM_CASE_SENSITIVE from _pytask.console import console from _pytask.console import create_summary_panel -from _pytask.console import format_task_id +from _pytask.console import format_task_name from _pytask.exceptions import CollectionError from _pytask.mark_utils import has_mark from _pytask.models import NodeInfo @@ -275,8 +275,8 @@ def pytask_collect_modify_tasks(tasks: list[PTask]) -> None: """Given all tasks, assign a short uniquely identifiable name to each task.""" id_to_short_id = _find_shortest_uniquely_identifiable_name_for_tasks(tasks) for task in tasks: - short_id = id_to_short_id[task.name] - task.short_name = short_id + if task.name in id_to_short_id and isinstance(task, Task): + task.display_name = id_to_short_id[task.name] def _find_shortest_uniquely_identifiable_name_for_tasks( @@ -292,7 +292,7 @@ def _find_shortest_uniquely_identifiable_name_for_tasks( id_to_short_id = {} # Make attempt to add up to twenty parts of the path to ensure uniqueness. - id_to_task = {task.name: task for task in tasks} + id_to_task = {task.name: task for task in tasks if isinstance(task, Task)} for n_parts in range(1, 20): dupl_id_to_short_id = { id_: "/".join(task.path.parts[-n_parts:]) + "::" + task.base_name @@ -336,7 +336,7 @@ def pytask_collect_log( header = "Error" else: if isinstance(report.node, PTask): - short_name = format_task_id( + short_name = format_task_name( report.node, editor_url_scheme="no_link" ) else: diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 341daf52e..96b7c013d 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -12,7 +12,7 @@ from _pytask.console import console from _pytask.console import create_url_style_for_path from _pytask.console import FILE_ICON -from _pytask.console import format_task_id +from _pytask.console import format_task_name from _pytask.console import PYTHON_ICON from _pytask.console import TASK_ICON from _pytask.exceptions import CollectionError @@ -191,7 +191,7 @@ def _print_collected_tasks( ) for task in tasks: - reduced_task_name = format_task_id( + reduced_task_name = format_task_name( task, editor_url_scheme=editor_url_scheme ) task_branch = module_branch.add( diff --git a/src/_pytask/console.py b/src/_pytask/console.py index 1b5b4f0c6..4ffd52d52 100644 --- a/src/_pytask/console.py +++ b/src/_pytask/console.py @@ -36,7 +36,7 @@ "create_url_style_for_task", "create_url_style_for_path", "console", - "format_task_id", + "format_task_name", "format_strings_as_flat_tree", "render_to_string", "unify_styles", @@ -143,7 +143,7 @@ def render_to_string( return rendered -def format_task_id(task: PTask, editor_url_scheme: str) -> Text: +def format_task_name(task: PTask, editor_url_scheme: str) -> Text: """Format a task id.""" if task.function is None: url_style = Style() @@ -151,12 +151,13 @@ def format_task_id(task: PTask, editor_url_scheme: str) -> Text: url_style = create_url_style_for_task(task.function, editor_url_scheme) if isinstance(task, Task): - path, task_name = task.short_name.split("::") + path, task_name = task.display_name.split("::") task_id = Text.assemble( Text(path + "::", style="dim"), Text(task_name, style=url_style) ) else: - task_id = Text(task.name, style=url_style) + name = getattr(task, "display_name", task.name) + task_id = Text(name, style=url_style) return task_id diff --git a/src/_pytask/dag_utils.py b/src/_pytask/dag_utils.py index aea3385bd..2f6da5d96 100644 --- a/src/_pytask/dag_utils.py +++ b/src/_pytask/dag_utils.py @@ -7,7 +7,7 @@ import networkx as nx from _pytask.console import format_strings_as_flat_tree -from _pytask.console import format_task_id +from _pytask.console import format_task_name from _pytask.console import TASK_ICON from _pytask.mark_utils import has_mark from _pytask.node_protocols import PTask @@ -162,7 +162,7 @@ def _extract_priorities_from_tasks(tasks: list[PTask]) -> dict[str, int]: name_to_task = {task.name: task for task in tasks} reduced_names = [] for name in tasks_w_mixed_priorities: - reduced_name = format_task_id(name_to_task[name], "no_link") + reduced_name = format_task_name(name_to_task[name], "no_link") reduced_names.append(reduced_name.plain) text = format_strings_as_flat_tree( diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index bc504e170..66b832203 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -11,7 +11,7 @@ from _pytask.console import create_summary_panel from _pytask.console import create_url_style_for_task from _pytask.console import format_strings_as_flat_tree -from _pytask.console import format_task_id +from _pytask.console import format_task_name from _pytask.console import unify_styles from _pytask.dag_utils import descending_tasks from _pytask.dag_utils import TopologicalSorter @@ -287,7 +287,7 @@ def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> def _print_errored_task_report(session: Session, report: ExecutionReport) -> None: """Print the traceback and the exception of an errored report.""" - task_name = format_task_id( + task_name = format_task_name( task=report.task, editor_url_scheme=session.config["editor_url_scheme"] ) text = Text.assemble("Task ", task_name, " failed", style="failed") diff --git a/src/_pytask/live.py b/src/_pytask/live.py index 6aeaa8e84..00ca9f301 100644 --- a/src/_pytask/live.py +++ b/src/_pytask/live.py @@ -8,7 +8,7 @@ import click from _pytask.config import hookimpl from _pytask.console import console -from _pytask.console import format_task_id +from _pytask.console import format_task_name from _pytask.node_protocols import PTask from _pytask.outcomes import CollectionOutcome from _pytask.outcomes import TaskOutcome @@ -223,12 +223,12 @@ def _generate_table( table.add_column("Outcome") for report in relevant_reports: table.add_row( - format_task_id(report.task, editor_url_scheme=self.editor_url_scheme), + format_task_name(report.task, editor_url_scheme=self.editor_url_scheme), Text(report.outcome.symbol, style=report.outcome.style), ) for task in self._running_tasks.values(): table.add_row( - format_task_id(task, editor_url_scheme=self.editor_url_scheme), + format_task_name(task, editor_url_scheme=self.editor_url_scheme), "running", ) @@ -260,7 +260,7 @@ def update_reports(self, new_report: ExecutionReport) -> None: self._running_tasks.pop(new_report.task.name) self._reports.append( _ReportEntry( - name=new_report.task.short_name, + name=getattr(new_report.task, "display_name", new_report.task.name), outcome=new_report.outcome, task=new_report.task, ) diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index fff2e2c7c..dbec41a53 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -46,7 +46,7 @@ class Task(MetaNode): """The task function.""" name: str | None = field(default=None, init=False) """The name of the task.""" - short_name: str | None = field(default=None, init=False) + display_name: str | None = field(default=None, init=False) """The shortest uniquely identifiable name for task for display.""" depends_on: PyTree[Node] = field(factory=dict) """A list of dependencies of task.""" @@ -64,8 +64,8 @@ def __attrs_post_init__(self: Task) -> None: if self.name is None: self.name = self.path.as_posix() + "::" + self.base_name - if self.short_name is None: - self.short_name = self.name + if self.display_name is None: + self.display_name = self.name def state(self) -> str | None: """Return the state of the node.""" diff --git a/src/_pytask/profile.py b/src/_pytask/profile.py index 9327a5cfc..ce7a18819 100644 --- a/src/_pytask/profile.py +++ b/src/_pytask/profile.py @@ -18,7 +18,7 @@ from _pytask.click import EnumChoice from _pytask.config import hookimpl from _pytask.console import console -from _pytask.console import format_task_id +from _pytask.console import format_task_name from _pytask.database_utils import BaseTable from _pytask.database_utils import DatabaseSession from _pytask.exceptions import CollectionError @@ -178,7 +178,7 @@ def _print_profile_table( table.add_column(name, justify="right") for task_name, info in profile.items(): - task_id = format_task_id( + task_id = format_task_name( task=name_to_task[task_name], editor_url_scheme=config["editor_url_scheme"], ) diff --git a/src/_pytask/shared.py b/src/_pytask/shared.py index 0ffaf722a..56dae2a22 100644 --- a/src/_pytask/shared.py +++ b/src/_pytask/shared.py @@ -9,7 +9,7 @@ import click import networkx as nx -from _pytask.console import format_task_id +from _pytask.console import format_task_name from _pytask.node_protocols import MetaNode from _pytask.node_protocols import PPathNode from _pytask.node_protocols import PTask @@ -92,7 +92,7 @@ def reduce_names_of_multiple_nodes( node = dag.nodes[name].get("node") or dag.nodes[name].get("task") if isinstance(node, PTask): - short_name = format_task_id(node, editor_url_scheme="no_link") + short_name = format_task_name(node, editor_url_scheme="no_link") elif isinstance(node, MetaNode): short_name = reduce_node_name(node, paths) else: diff --git a/src/_pytask/warnings_utils.py b/src/_pytask/warnings_utils.py index 739a49ab1..984bfe15f 100644 --- a/src/_pytask/warnings_utils.py +++ b/src/_pytask/warnings_utils.py @@ -173,10 +173,7 @@ def catch_warnings_for_item( yield - if task is not None: - id_ = task.short_name if hasattr(task, "short_name") else task.name - else: - id_ = when + id_ = getattr(task, "display_name", task.name) if task is not None else when for warning_message in log: fs_location = warning_message.filename, warning_message.lineno diff --git a/tests/test_console.py b/tests/test_console.py index ea8f86714..f4ba00485 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -9,7 +9,7 @@ from _pytask.console import create_summary_panel from _pytask.console import create_url_style_for_path from _pytask.console import create_url_style_for_task -from _pytask.console import format_task_id +from _pytask.console import format_task_name from _pytask.console import render_to_string from pytask import CollectionOutcome from pytask import console @@ -139,9 +139,9 @@ def test_format_task_id( task = Task(base_name=base_name, path=path, function=task_func) if short_name is not None: - task.short_name = short_name + task.display_name = short_name - result = format_task_id(task, editor_url_scheme) + result = format_task_name(task, editor_url_scheme) assert result == expected diff --git a/tests/test_live.py b/tests/test_live.py index 4fe76273c..a878c6458 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -30,7 +30,7 @@ def test_verbose_mode_execution(tmp_path, runner, verbose): def test_live_execution_sequentially(capsys, tmp_path): path = tmp_path.joinpath("task_module.py") task = Task(base_name="task_example", path=path, function=lambda x: x) - task.short_name = "task_module.py::task_example" + task.display_name = "task_module.py::task_example" live_manager = LiveManager() live = LiveExecution( @@ -88,7 +88,7 @@ def test_live_execution_sequentially(capsys, tmp_path): def test_live_execution_displays_skips_and_persists(capsys, tmp_path, verbose, outcome): path = tmp_path.joinpath("task_module.py") task = Task(base_name="task_example", path=path, function=lambda x: x) - task.short_name = "task_module.py::task_example" + task.display_name = "task_module.py::task_example" live_manager = LiveManager() live = LiveExecution( @@ -137,7 +137,7 @@ def test_live_execution_displays_skips_and_persists(capsys, tmp_path, verbose, o def test_live_execution_displays_subset_of_table(capsys, tmp_path, n_entries_in_table): path = tmp_path.joinpath("task_module.py") running_task = Task(base_name="task_running", path=path, function=lambda x: x) - running_task.short_name = "task_module.py::task_running" + running_task.display_name = "task_module.py::task_running" live_manager = LiveManager() live = LiveExecution( @@ -160,7 +160,7 @@ def test_live_execution_displays_subset_of_table(capsys, tmp_path, n_entries_in_ assert "Completed: 0/2" in captured.out completed_task = Task(base_name="task_completed", path=path, function=lambda x: x) - completed_task.short_name = "task_module.py::task_completed" + completed_task.display_name = "task_module.py::task_completed" live.update_running_tasks(completed_task) report = ExecutionReport( task=completed_task, outcome=TaskOutcome.SUCCESS, exc_info=None @@ -191,7 +191,7 @@ def test_live_execution_displays_subset_of_table(capsys, tmp_path, n_entries_in_ def test_live_execution_skips_do_not_crowd_out_displayed_tasks(capsys, tmp_path): path = tmp_path.joinpath("task_module.py") task = Task(base_name="task_example", path=path, function=lambda x: x) - task.short_name = "task_module.py::task_example" + task.display_name = "task_module.py::task_example" live_manager = LiveManager() live = LiveExecution( @@ -215,12 +215,12 @@ def test_live_execution_skips_do_not_crowd_out_displayed_tasks(capsys, tmp_path) # Add one displayed reports and many more not displayed reports to crowd out the # valid one. successful_task = Task(base_name="task_success", path=path, function=lambda x: x) - successful_task.short_name = "task_module.py::task_success" + successful_task.display_name = "task_module.py::task_success" tasks = [] for i in range(25): skipped_task = Task(base_name=f"task_skip_{i}", path=path, function=lambda x: x) - skipped_task.short_name = f"task_module.py::task_skip_{i}" + skipped_task.display_name = f"task_module.py::task_skip_{i}" tasks.append(skipped_task) live_manager.start() From 11616602d46e5d6e239a9edae858a5ce0f22b67b Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Mon, 4 Sep 2023 14:00:41 +0200 Subject: [PATCH 06/10] Filter non-taskwpath based tasks. --- src/_pytask/clean.py | 4 +++- src/_pytask/collect_command.py | 16 +++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/_pytask/clean.py b/src/_pytask/clean.py index 295a291a2..41e128f89 100644 --- a/src/_pytask/clean.py +++ b/src/_pytask/clean.py @@ -23,6 +23,7 @@ from _pytask.git import is_git_installed from _pytask.node_protocols import PPathNode from _pytask.node_protocols import PTask +from _pytask.node_protocols import PTaskWithPath from _pytask.outcomes import ExitCode from _pytask.path import find_common_ancestor from _pytask.path import relative_to @@ -217,7 +218,8 @@ def _collect_all_paths_known_to_pytask(session: Session) -> set[Path]: def _yield_paths_from_task(task: PTask) -> Generator[Path, None, None]: """Yield all paths attached to a task.""" - yield task.path + if isinstance(task, PTaskWithPath): + yield task.path for attribute in ("depends_on", "produces"): for node in tree_leaves(getattr(task, attribute)): if isinstance(node, PPathNode): diff --git a/src/_pytask/collect_command.py b/src/_pytask/collect_command.py index 96b7c013d..1dfac4b8a 100644 --- a/src/_pytask/collect_command.py +++ b/src/_pytask/collect_command.py @@ -2,6 +2,7 @@ from __future__ import annotations import sys +from collections import defaultdict from pathlib import Path from typing import Any from typing import TYPE_CHECKING @@ -22,6 +23,7 @@ from _pytask.mark import select_by_mark from _pytask.node_protocols import PPathNode from _pytask.node_protocols import PTask +from _pytask.node_protocols import PTaskWithPath from _pytask.outcomes import ExitCode from _pytask.path import find_common_ancestor from _pytask.path import relative_to @@ -76,11 +78,12 @@ def collect(**raw_config: Any | None) -> NoReturn: session.hook.pytask_dag(session=session) tasks = _select_tasks_by_expressions_and_marker(session) + task_with_path = [t for t in tasks if isinstance(t, PTaskWithPath)] common_ancestor = _find_common_ancestor_of_all_nodes( - tasks, session.config["paths"], session.config["nodes"] + task_with_path, session.config["paths"], session.config["nodes"] ) - dictionary = _organize_tasks(tasks) + dictionary = _organize_tasks(task_with_path) if dictionary: _print_collected_tasks( dictionary, @@ -117,7 +120,7 @@ def _select_tasks_by_expressions_and_marker(session: Session) -> list[PTask]: def _find_common_ancestor_of_all_nodes( - tasks: list[PTask], paths: list[Path], show_nodes: bool + tasks: list[PTaskWithPath], paths: list[Path], show_nodes: bool ) -> Path: """Find common ancestor from all nodes and passed paths.""" all_paths = [] @@ -136,16 +139,15 @@ def _find_common_ancestor_of_all_nodes( return common_ancestor -def _organize_tasks(tasks: list[PTask]) -> dict[Path, list[PTask]]: +def _organize_tasks(tasks: list[PTaskWithPath]) -> dict[Path, list[PTaskWithPath]]: """Organize tasks in a dictionary. The dictionary has file names as keys and then a dictionary with task names and below a dictionary with dependencies and targets. """ - dictionary: dict[Path, list[PTask]] = {} + dictionary: dict[Path, list[PTaskWithPath]] = defaultdict(list) for task in tasks: - dictionary[task.path] = dictionary.get(task.path, []) dictionary[task.path].append(task) sorted_dict = {} @@ -156,7 +158,7 @@ def _organize_tasks(tasks: list[PTask]) -> dict[Path, list[PTask]]: def _print_collected_tasks( - dictionary: dict[Path, list[PTask]], + dictionary: dict[Path, list[PTaskWithPath]], show_nodes: bool, editor_url_scheme: str, common_ancestor: Path, From c21f1397e3f20b043f97d75892536b3ff9d2b8bc Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Mon, 4 Sep 2023 14:04:35 +0200 Subject: [PATCH 07/10] Fix. --- src/_pytask/dag.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py index 753725842..e3d0ccb5a 100644 --- a/src/_pytask/dag.py +++ b/src/_pytask/dag.py @@ -34,7 +34,6 @@ from _pytask.shared import reduce_node_name from _pytask.traceback import render_exc_info from _pytask.tree_util import tree_map -from pytask import Task from rich.text import Text from rich.tree import Tree @@ -144,8 +143,7 @@ def pytask_dag_has_node_changed(node: MetaNode, task_name: str) -> bool: if db_state is None: return True - # TODO: Could be MetaNode with Path? - if isinstance(node, (PPathNode, Task)): + if isinstance(node, (PPathNode, PTaskWithPath)): # If the modification times match, the node has not been changed. if node_state == db_state.modification_time: return False From 019099db8b9aa3e9277b3b21b346e0f144ee7e49 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Mon, 4 Sep 2023 14:07:58 +0200 Subject: [PATCH 08/10] Refactored from task to pytask_execute. --- src/_pytask/execute.py | 21 ++++++++++++++++++++- src/_pytask/nodes.py | 20 +------------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index 66b832203..2a473f015 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -33,7 +33,9 @@ from _pytask.traceback import format_exception_without_traceback from _pytask.traceback import remove_traceback_from_exc_info from _pytask.traceback import render_exc_info +from _pytask.tree_util import tree_leaves from _pytask.tree_util import tree_map +from _pytask.tree_util import tree_structure from rich.text import Text @@ -153,7 +155,24 @@ def pytask_execute_task(session: Session, task: PTask) -> bool: if name in parameters: kwargs[name] = tree_map(lambda x: x.load(), value) - task.execute(**kwargs) + out = task.execute(**kwargs) + + if "return" in task.produces: + structure_out = tree_structure(out) + structure_return = tree_structure(task.produces["return"]) + # strict must be false when none is leaf. + if not structure_return.is_prefix(structure_out, strict=False): + raise ValueError( + "The structure of the return annotation is not a subtree of the " + "structure of the function return.\n\nFunction return: " + f"{structure_out}\n\nReturn annotation: {structure_return}" + ) + + nodes = tree_leaves(task.produces["return"]) + values = structure_return.flatten_up_to(out) + for node, value in zip(nodes, values): + node.save(value) + return True diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index dbec41a53..5d4b493b4 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -12,8 +12,6 @@ from _pytask.node_protocols import Node from _pytask.node_protocols import PPathNode from _pytask.tree_util import PyTree -from _pytask.tree_util import tree_leaves -from _pytask.tree_util import tree_structure from attrs import define from attrs import field @@ -76,23 +74,7 @@ def state(self) -> str | None: def execute(self, **kwargs: Any) -> None: """Execute the task.""" out = self.function(**kwargs) - - # TODO: Move to pytask_execute_task. - if "return" in self.produces: - structure_out = tree_structure(out) - structure_return = tree_structure(self.produces["return"]) - # strict must be false when none is leaf. - if not structure_return.is_prefix(structure_out, strict=False): - raise ValueError( - "The structure of the return annotation is not a subtree of the " - "structure of the function return.\n\nFunction return: " - f"{structure_out}\n\nReturn annotation: {structure_return}" - ) - - nodes = tree_leaves(self.produces["return"]) - values = structure_return.flatten_up_to(out) - for node, value in zip(nodes, values): - node.save(value) + return out def add_report_section(self, when: str, key: str, content: str) -> None: """Add sections which will be displayed in report like stdout or stderr.""" From 6e377e33486f482948b5e23fe633a7dab592904c Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Mon, 4 Sep 2023 14:16:02 +0200 Subject: [PATCH 09/10] Remove add_report_section. --- src/_pytask/capture.py | 6 ++++-- src/_pytask/node_protocols.py | 6 +----- src/_pytask/nodes.py | 8 +------- src/_pytask/report.py | 4 ++-- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index 01a9efa8f..cc8886c7b 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -716,8 +716,10 @@ def task_capture(self, when: str, task: PTask) -> Generator[None, None, None]: self.suspend(in_=False) out, err = self.read() - task.add_report_section(when, "stdout", out) - task.add_report_section(when, "stderr", err) + if out: + task.report_sections.append((when, "stdout", out)) + if err: + task.report_sections.append((when, "stderr", err)) # Hooks diff --git a/src/_pytask/node_protocols.py b/src/_pytask/node_protocols.py index ee992b8ae..f0463dc41 100644 --- a/src/_pytask/node_protocols.py +++ b/src/_pytask/node_protocols.py @@ -64,7 +64,7 @@ class PTask(MetaNode, Protocol): depends_on: PyTree[Node] produces: PyTree[Node] markers: list[Mark] - _report_sections: list[tuple[str, str, str]] + report_sections: list[tuple[str, str, str]] attributes: dict[Any, Any] function: Callable[..., Any] @@ -72,10 +72,6 @@ def execute(self, **kwargs: Any) -> Any: """Return the value of the node that will be injected into the task.""" ... - def add_report_section(self, when: str, key: str, content: str) -> None: - """Add sections which will be displayed in report like stdout and stderr.""" - ... - @runtime_checkable class PTaskWithPath(PTask, Protocol): diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index 5d4b493b4..8275a0251 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -52,7 +52,7 @@ class Task(MetaNode): """A list of products of task.""" markers: list[Mark] = field(factory=list) """A list of markers attached to the task function.""" - _report_sections: list[tuple[str, str, str]] = field(factory=list) + report_sections: list[tuple[str, str, str]] = field(factory=list) """Reports with entries for when, what, and content.""" attributes: dict[Any, Any] = field(factory=dict) """A dictionary to store additional information of the task.""" @@ -76,12 +76,6 @@ def execute(self, **kwargs: Any) -> None: out = self.function(**kwargs) return out - def add_report_section(self, when: str, key: str, content: str) -> None: - """Add sections which will be displayed in report like stdout or stderr.""" - # TODO: Remove method. - if content: - self._report_sections.append((when, key, content)) - @define(kw_only=True) class PathNode(PPathNode): diff --git a/src/_pytask/report.py b/src/_pytask/report.py index dd9882226..1f597f798 100644 --- a/src/_pytask/report.py +++ b/src/_pytask/report.py @@ -68,9 +68,9 @@ def from_task_and_exception( ) -> ExecutionReport: """Create a report from a task and an exception.""" exc_info = remove_internal_traceback_frames_from_exc_info(exc_info) - return cls(task, TaskOutcome.FAIL, exc_info, task._report_sections) + return cls(task, TaskOutcome.FAIL, exc_info, task.report_sections) @classmethod def from_task(cls, task: PTask) -> ExecutionReport: """Create a report from a task.""" - return cls(task, TaskOutcome.SUCCESS, None, task._report_sections) + return cls(task, TaskOutcome.SUCCESS, None, task.report_sections) From e451a33e03443d293476f1a393f61b1b43ab3c16 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Mon, 4 Sep 2023 14:19:33 +0200 Subject: [PATCH 10/10] Add tasks. --- docs/source/changes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/changes.md b/docs/source/changes.md index eaa1292d0..6fd4a1dee 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -28,6 +28,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`408` removes `.value` from `Node` protocol. - {pull}`409` make `.from_annot` an optional feature of nodes. - {pull}`410` allows to pass functions to `PythonNode(hash=...)`. +- {pull}`412` adds protocols for tasks. ## 0.3.2 - 2023-06-07