diff --git a/README.rst b/README.rst index c7c1ca3f4..447952a48 100644 --- a/README.rst +++ b/README.rst @@ -60,6 +60,10 @@ projects. Its features include: `_ if a task fails, get feedback quickly, and be more productive. +- **Parametrizations via loops.** `Loop over task functions + `_ + to run the same task with different inputs. + - **Select tasks via expressions.** Run only a subset of tasks with `expressions and marker expressions `_ diff --git a/docs/source/_static/images/how-to-parametrize-a-task.png b/docs/source/_static/images/how-to-parametrize-a-task.png new file mode 100644 index 000000000..ec2e8bd82 Binary files /dev/null and b/docs/source/_static/images/how-to-parametrize-a-task.png differ diff --git a/docs/source/changes.rst b/docs/source/changes.rst index 97291edbd..cd66b140d 100644 --- a/docs/source/changes.rst +++ b/docs/source/changes.rst @@ -16,6 +16,8 @@ all releases are available on `PyPI `_ and parametrized arguments to the task class. - :pull:`228` removes ``task.pytaskmark`` and moves the information to :attr:`_pytask.models.CollectionMetadata.markers`. +- :pull:`229` implements a new loop-based approach to parametrizations using the + :func:`@pytask.mark.task <_pytask.task.task>` decorator. - :pull:`230` implements :class:`_pytask.logging._TimeUnit` as a :class:`typing.NamedTuple` for better typing. diff --git a/docs/source/how_to_guides/bp_parametrizations.rst b/docs/source/how_to_guides/bp_parametrizations.rst index 6dd8728b8..5602ef4b6 100644 --- a/docs/source/how_to_guides/bp_parametrizations.rst +++ b/docs/source/how_to_guides/bp_parametrizations.rst @@ -106,26 +106,27 @@ parametrization. def _create_parametrization(data): - parametrizations = [] - ids = [] + id_to_kwargs = {} for data_name in data: - ids.append(data_name) depends_on = path_to_input_data(data_name) produces = path_to_processed_data(data_name) - parametrizations.append((depends_on, produces)) - return "depends_on, produces", parametrizations, ids + id_to_kwargs[data_name] = {"depends_on": depends_on, "produces": produces} + return id_to_kwargs - _SIGNATURE, _PARAMETRIZATION, _IDS = _create_parametrization(DATA) + _ID_TO_KWARGS = _create_parametrization(DATA) - @pytask.mark.parametrize(_SIGNATURE, _PARAMETRIZATION, ids=_IDS) - def task_prepare_data(depends_on, produces): - ... + for id_, kwargs in _ID_TO_KWARGS.items(): -All arguments for the ``parametrize`` decorator are built within a function to keep the -logic in one place and the namespace of the module clean. + @pytask.mark.task(id=id_, kwargs=kwargs) + def task_prepare_data(depends_on, produces): + ... + +All arguments for the loop and the :func:`@pytask.mark.task <_pytask.task_utils.task>` +decorator are built within a function to keep the logic in one place and the namespace +of the module clean. Ids are used to make the task :ref:`ids ` more descriptive and to simplify their selection with :ref:`expressions `. Here is an example of the task ids with @@ -183,25 +184,29 @@ And, here is the task file. def _create_parametrization(estimations): - parametrizations = [] - ids = [] + id_to_kwargs = {} for name, config in estimations.items(): - ids.append(name) depends_on = path_to_processed_data(config["data"]) produces = path_to_estimation_result(name) - parametrizations.append((depends_on, config["model"], produces)) - return "depends_on, model, produces", parametrizations, ids + id_to_kwargs[name] = { + "depends_on": depends_on, + "model": config["model"], + "produces": produces, + } + return id_to_kwargs - _SIGNATURE, _PARAMETRIZATION, _IDS = _create_parametrization(ESTIMATIONS) + _ID_TO_KWARGS = _create_parametrization(ESTIMATIONS) - @pytask.mark.parametrize(_SIGNATURE, _PARAMETRIZATION, ids=_IDS) - def task_estmate_models(depends_on, model, produces): - if model == "linear_probability": - ... - ... + + for id_, kwargs in _ID_TO_KWARGS.items(): + + @pytask.mark.task(id=id_, kwargs=kwars) + def task_estmate_models(depends_on, model, produces): + if model == "linear_probability": + ... Replicating this pattern across a project allows for a clean way to define parametrizations. diff --git a/docs/source/how_to_guides/how_to_parametrize_a_task_the_pytest_way.rst b/docs/source/how_to_guides/how_to_parametrize_a_task_the_pytest_way.rst new file mode 100644 index 000000000..e98a440e1 --- /dev/null +++ b/docs/source/how_to_guides/how_to_parametrize_a_task_the_pytest_way.rst @@ -0,0 +1,225 @@ +How to parametrize a task - The pytest way +========================================== + +You want to define a task which should be repeated over a range of inputs? Parametrize +your task function! + +.. important:: + + This guide shows you how to parametrize tasks with the pytest approach. For the new + and preferred approach, see this :doc:`tutorial + <../tutorials/how_to_parametrize_a_task>`. + +You want to define a task which should be repeated over a range of inputs? Parametrize +your task function! + +.. seealso:: + + If you want to know more about best practices for parametrizations, check out this + :doc:`guide <../how_to_guides/bp_parametrizations>` after you made yourself familiar + this tutorial. + + +An example +---------- + +We reuse the previous example of a task which generates random data and repeat the same +operation over a number of seeds to receive multiple, reproducible samples. + +First, we write the task for one seed. + +.. code-block:: python + + import numpy as np + import pytask + + + @pytask.mark.produces(BLD / "data_0.pkl") + def task_create_random_data(produces): + rng = np.random.default_rng(0) + ... + +In the next step, we repeat the same task over the numbers 0, 1, and 2 and pass them to +the ``seed`` argument. We also vary the name of the produced file in every iteration. + +.. code-block:: python + + @pytask.mark.parametrize( + "produces, seed", + [(BLD / "data_0.pkl", 0), (BLD / "data_1.pkl", 1), (BLD / "data_2.pkl", 2)], + ) + def task_create_random_data(seed, produces): + rng = np.random.default_rng(seed) + ... + +The parametrize decorator receives two arguments. The first argument is ``"produces, +seed"`` - the signature. It is a comma-separated string where each value specifies the +name of a task function argument. + +.. seealso:: + + The signature is explained in detail :ref:`below `. + +The second argument of the parametrize decorator is a list (or any iterable) which has +as many elements as there are iterations over the task function. Each element has to +provide one value for each argument name in the signature - two in this case. + +Putting all together, the task is executed three times and each run the path from the +list is mapped to the argument ``produces`` and ``seed`` receives the seed. + +.. note:: + + If you use ``produces`` or ``depends_on`` in the signature of the parametrize + decorator, the values are handled as if they were attached to the function with + ``@pytask.mark.depends_on`` or ``@pytask.mark.produces``. + +Un-parametrized dependencies +---------------------------- + +To specify a dependency which is the same for all parametrizations, add it with +``pytask.mark.depends_on``. + +.. code-block:: python + + @pytask.mark.depends_on(SRC / "common_dependency.file") + @pytask.mark.parametrize( + "produces, seed", + [(BLD / "data_0.pkl", 0), (BLD / "data_1.pkl", 1), (BLD / "data_2.pkl", 2)], + ) + def task_create_random_data(seed, produces): + rng = np.random.default_rng(seed) + ... + + +.. _parametrize_signature: + +The signature +------------- + +The signature can be passed in three different formats. + +1. The signature can be a comma-separated string like an entry in a csv table. Note that + white-space is stripped from each name which you can use to separate the names for + readability. Here are some examples: + + .. code-block:: python + + "single_argument" + "first_argument,second_argument" + "first_argument, second_argument" + +2. The signature can be a tuple of strings where each string is one argument name. Here + is an example. + + .. code-block:: python + + ("first_argument", "second_argument") + +3. Finally, it is also possible to use a list of strings. + + .. code-block:: python + + ["first_argument", "second_argument"] + + +The id +------ + +Every task has a unique id which can be used to :doc:`select it `. +The normal id combines the path to the module where the task is defined, a double colon, +and the name of the task function. Here is an example. + +.. code-block:: + + ../task_example.py::task_example + +This behavior would produce duplicate ids for parametrized tasks. Therefore, there exist +multiple mechanisms to produce unique ids. + + +.. _auto_generated_ids: + +Auto-generated ids +~~~~~~~~~~~~~~~~~~ + +To avoid duplicate task ids, the ids of parametrized tasks are extended with +descriptions of the values they are parametrized with. Booleans, floats, integers and +strings enter the task id directly. For example, a task function which receives four +arguments, ``True``, ``1.0``, ``2``, and ``"hello"``, one of each dtype, has the +following id. + +.. code-block:: + + task_example.py::task_example[True-1.0-2-hello] + +Arguments with other dtypes cannot be easily converted to strings and, thus, are +replaced with a combination of the argument name and the iteration counter. + +For example, the following function is parametrized with tuples. + +.. code-block:: python + + @pytask.mark.parametrized("i", [(0,), (1,)]) + def task_example(i): + pass + +Since the tuples are not converted to strings, the ids of the two tasks are + +.. code-block:: + + task_example.py::task_example[i0] + task_example.py::task_example[i1] + + +User-defined ids +~~~~~~~~~~~~~~~~ + +Instead of a function, you can also pass a list or another iterable of id values via +``ids``. + +This code + +.. code-block:: python + + @pytask.mark.parametrized("i", [(0,), (1,)], ids=["first", "second"]) + def task_example(i): + pass + +produces these ids + +.. code-block:: + + task_example.py::task_example[first] # (0,) + task_example.py::task_example[second] # (1,) + + +.. _how_to_parametrize_a_task_convert_other_objects: + +Convert other objects +~~~~~~~~~~~~~~~~~~~~~ + +To change the representation of tuples and other objects, you can pass a function to the +``ids`` argument of the :func:`@pytask.mark.parametrize +<_pytask.parametrize.parametrize>` decorator. The function is called for every argument +and may return a boolean, number, or string which will be integrated into the id. For +every other return, the auto-generated value is used. + +To get a unique representation of a tuple, we can use the hash value. + +.. code-block:: python + + def tuple_to_hash(value): + if isinstance(value, tuple): + return hash(a) + + + @pytask.mark.parametrized("i", [(0,), (1,)], ids=tuple_to_hash) + def task_example(i): + pass + +This produces the following ids: + +.. code-block:: + + task_example.py::task_example[3430018387555] # (0,) + task_example.py::task_example[3430019387558] # (1,) diff --git a/docs/source/how_to_guides/index.rst b/docs/source/how_to_guides/index.rst index e2e9cf0f2..39e9da6cc 100644 --- a/docs/source/how_to_guides/index.rst +++ b/docs/source/how_to_guides/index.rst @@ -15,6 +15,7 @@ specific tasks with pytask. how_to_write_a_plugin how_to_influence_build_order + how_to_parametrize_a_task_the_pytest_way Best Practice Guides diff --git a/docs/source/tutorials/how_to_parametrize_a_task.rst b/docs/source/tutorials/how_to_parametrize_a_task.rst index 53cd0f9e9..5e28b35af 100644 --- a/docs/source/tutorials/how_to_parametrize_a_task.rst +++ b/docs/source/tutorials/how_to_parametrize_a_task.rst @@ -1,23 +1,29 @@ How to parametrize a task ========================= -Often, you want to define a task which should be repeated over a range of inputs. pytask -allows to parametrize task functions to accomplish this behavior. +You want to define a task which should be repeated over a range of inputs? Parametrize +your task function! -.. seealso:: +.. important:: - If you want to know more about best practices for parametrizations, check out this - :doc:`guide <../how_to_guides/bp_parametrizations>` after you made yourself familiar - this tutorial. + Before v0.2.0, pytask supported only one approach to parametrizations which is + similar to pytest's using a :func:`@pytask.mark.parametrize + <_pytask.parametrize.parametrize>` decorator. You can find it :doc:`here + <../how_to_guides/how_to_parametrize_a_task_the_pytest_way>`. + + Here you find the new and preferred approach. An example ---------- -We reuse the previous example of a task which generates random data and repeat the same -operation over a number of seeds to receive multiple, reproducible samples. +We reuse the task from the previous :doc:`tutorial ` which +generates random data and repeat the same operation over a number of seeds to receive +multiple, reproducible samples. -First, we write the task for one seed. +Apply the :func:`@pytask.mark.task <_pytask.task_utils.task>` decorator, loop over the +function and supply different seeds and output paths as default arguments of the +function. .. code-block:: python @@ -25,92 +31,36 @@ First, we write the task for one seed. import pytask - @pytask.mark.produces(BLD / "data_0.pkl") - def task_create_random_data(produces): - rng = np.random.default_rng(0) - ... - -In the next step, we repeat the same task over the numbers 0, 1, and 2 and pass them to -the ``seed`` argument. We also vary the name of the produced file in every iteration. - -.. code-block:: python - - @pytask.mark.parametrize( - "produces, seed", - [(BLD / "data_0.pkl", 0), (BLD / "data_1.pkl", 1), (BLD / "data_2.pkl", 2)], - ) - def task_create_random_data(seed, produces): - rng = np.random.default_rng(seed) - ... - -The parametrize decorator receives two arguments. The first argument is ``"produces, -seed"`` - the signature. It is a comma-separated string where each value specifies the -name of a task function argument. - -.. seealso:: + for i in range(10): - The signature is explained in detail :ref:`below `. + @pytask.mark.task + def task_create_random_data(produces=f"data_{i}.pkl", seed=i): + rng = np.random.default_rng(seed) + ... -The second argument of the parametrize decorator is a list (or any iterable) which has -as many elements as there are iterations over the task function. Each element has to -provide one value for each argument name in the signature - two in this case. +Executing pytask gives you this: -Putting all together, the task is executed three times and each run the path from the -list is mapped to the argument ``produces`` and ``seed`` receives the seed. +.. image:: /_static/images/how-to-parametrize-a-task.png -.. note:: - If you use ``produces`` or ``depends_on`` in the signature of the parametrize - decorator, the values are handled as if they were attached to the function with - ``@pytask.mark.depends_on`` or ``@pytask.mark.produces``. +``depends_on`` and ``produces`` +------------------------------- -Un-parametrized dependencies ----------------------------- +You can also use decorators to supply values to the function. To specify a dependency which is the same for all parametrizations, add it with -``pytask.mark.depends_on``. +``@pytask.mark.depends_on``. And add a product with ``@pytask.mark.produces`` .. code-block:: python - @pytask.mark.depends_on(SRC / "common_dependency.file") - @pytask.mark.parametrize( - "produces, seed", - [(BLD / "data_0.pkl", 0), (BLD / "data_1.pkl", 1), (BLD / "data_2.pkl", 2)], - ) - def task_create_random_data(seed, produces): - rng = np.random.default_rng(seed) - ... - - -.. _parametrize_signature: - -The signature -------------- + for i in range(10): -The signature can be passed in three different formats. - -1. The signature can be a comma-separated string like an entry in a csv table. Note that - white-space is stripped from each name which you can use to separate the names for - readability. Here are some examples: - - .. code-block:: python - - "single_argument" - "first_argument,second_argument" - "first_argument, second_argument" - -2. The signature can be a tuple of strings where each string is one argument name. Here - is an example. - - .. code-block:: python - - ("first_argument", "second_argument") - -3. Finally, it is also possible to use a list of strings. - - .. code-block:: python - - ["first_argument", "second_argument"] + @pytask.mark.task + @pytask.mark.depends_on(SRC / "common_dependency.file") + @pytask.mark.produces(f"data_{i}.pkl") + def task_create_random_data(produces, seed=i): + rng = np.random.default_rng(seed) + ... .. _how_to_parametrize_a_task_the_id: @@ -124,97 +74,105 @@ and the name of the task function. Here is an example. .. code-block:: - ../task_example.py::task_example - -This behavior would produce duplicate ids for parametrized tasks. Therefore, there exist -multiple mechanisms to produce unique ids. + ../task_data_preparation.py::task_create_random_data +This behavior would produce duplicate ids for parametrized tasks. By default, +auto-generated ids are used which are explained :ref:`here `. -Auto-generated ids -~~~~~~~~~~~~~~~~~~ +More powerful are user-defined ids. -To avoid duplicate task ids, the ids of parametrized tasks are extended with -descriptions of the values they are parametrized with. Booleans, floats, integers and -strings enter the task id directly. For example, a task function which receives four -arguments, ``True``, ``1.0``, ``2``, and ``"hello"``, one of each dtype, has the -following id. -.. code-block:: - - task_example.py::task_example[True-1.0-2-hello] +.. _ids: -Arguments with other dtypes cannot be easily converted to strings and, thus, are -replaced with a combination of the argument name and the iteration counter. +User-defined ids +~~~~~~~~~~~~~~~~ -For example, the following function is parametrized with tuples. +The :func:`@pytask.mark.task <_pytask.task_utils.task>` decorator has an ``id`` keyword +which allows the user to set the a special name for the iteration. .. code-block:: python - @pytask.mark.parametrized("i", [(0,), (1,)]) - def task_example(i): - pass + for seed, id_ in [(0, "first"), (1, "second")]: -Since the tuples are not converted to strings, the ids of the two tasks are + @pytask.mark.task(id=id_) + def task_create_random_data(seed=i, produces=f"out_{i}.txt"): + ... -.. code-block:: +produces these ids - task_example.py::task_example[i0] - task_example.py::task_example[i1] +.. code-block:: + task_data_preparation.py::task_create_random_data[first] + task_data_preparation.py::task_create_random_data[second] -.. _how_to_parametrize_a_task_convert_other_objects: -Convert other objects -~~~~~~~~~~~~~~~~~~~~~ +Complex example +--------------- -To change the representation of tuples and other objects, you can pass a function to the -``ids`` argument of the :func:`~_pytask.parametrize.parametrize` decorator. The function -is called for every argument and may return a boolean, number, or string which will be -integrated into the id. For every other return, the auto-generated value is used. +Parametrizations are becoming more complex quickly. Often, you need to supply many +arguments and ids to tasks. -To get a unique representation of a tuple, we can use the hash value. +To organize your ids and arguments use nested dictionaries where keys are ids and values +are dictionaries mapping from argument names to values. .. code-block:: python - def tuple_to_hash(value): - if isinstance(value, tuple): - return hash(a) + ID_TO_KWARGS = { + "first": { + "seed": 0, + "produces": "data_0.pkl", + }, + "second": { + "seed": 1, + "produces": "data_1.pkl", + }, + } +The parametrization becomes - @pytask.mark.parametrized("i", [(0,), (1,)], ids=tuple_to_hash) - def task_example(i): - pass - -This produces the following ids: +.. code-block:: python -.. code-block:: + for id_, kwargs in ID_TO_KWARGS.items(): - task_example.py::task_example[3430018387555] # (0,) - task_example.py::task_example[3430019387558] # (1,) + @pytask.mark.task(id=id_) + def task_create_random_data(seed=kwargs["seed"], produces=kwargs["produces"]): + ... +Unpacking all the arguments can become tedious. Use instead the ``kwargs`` argument of +the :func:`@pytask.mark.task <_pytask.task_utils.task` decorator to pass keyword +arguments to the task. -.. _ids: +.. code-block:: python -User-defined ids -~~~~~~~~~~~~~~~~ + for id_, kwargs in ID_TO_KWARGS.items(): -Instead of a function, you can also pass a list or another iterable of id values via -``ids``. + @pytask.mark.task(id=id_, kwargs=kwargs) + def task_create_random_data(seed, produces): + ... -This code +As a last step to organize our code even more, we can write a function which creates +``ID_TO_KWARGS``. You can hide the creation of input and output paths and other +arguments in this function. .. code-block:: python - @pytask.mark.parametrized("i", [(0,), (1,)], ids=["first", "second"]) - def task_example(i): - pass + def create_parametrization(): + id_to_kwargs = {} + for i, id_ in enumerate(["first", "second"]): + id_to_kwargs[id_] = {"produces": f"out_{i}.txt"} -produces these ids + return id_to_kwargs -.. code-block:: - task_example.py::task_example[first] # (0,) - task_example.py::task_example[second] # (1,) + ID_TO_KWARGS = create_parametrization() + + + for id_, kwargs in ID_TO_KWARGS.items(): + + @pytask.mark.task(id=id_, kwargs=kwargs) + def task_create_random_data(i, produces): + ... -This is arguably the easiest way to change the representation of many objects at once -while also producing ids which are easy to remember and type. +The :doc:`best-practices guide on parametrizations +<../how_to_guides/bp_parametrizations>` goes into even more detail on how to scale +parametrizations. diff --git a/docs/source/tutorials/how_to_write_a_task.rst b/docs/source/tutorials/how_to_write_a_task.rst index a8e0eba0e..7e6e7500a 100644 --- a/docs/source/tutorials/how_to_write_a_task.rst +++ b/docs/source/tutorials/how_to_write_a_task.rst @@ -43,9 +43,11 @@ Here, we define the function @pytask.mark.produces(BLD / "data.pkl") def task_create_random_data(produces): + rng = np.random.default_rng(0) beta = 2 - x = np.random.normal(loc=5, scale=10, size=1_000) - epsilon = np.random.standard_normal(1_000) + + x = rng.normal(loc=5, scale=10, size=1_000) + epsilon = rng.standard_normal(1_000) y = beta * x + epsilon diff --git a/environment.yml b/environment.yml index 5e78ecbf2..ddb22a611 100644 --- a/environment.yml +++ b/environment.yml @@ -36,6 +36,7 @@ dependencies: - pytest - pytest-cov - pytest-xdist + - tabulate - tox-conda # Documentation diff --git a/src/_pytask/__init__.py b/src/_pytask/__init__.py index f493641cf..8ee216253 100644 --- a/src/_pytask/__init__.py +++ b/src/_pytask/__init__.py @@ -1,3 +1,4 @@ +"""This module should not contain any imports except for the version.""" from __future__ import annotations try: diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index d56b5f282..55d0c5202 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -2,6 +2,7 @@ from __future__ import annotations import inspect +import itertools import os import sys import time @@ -65,7 +66,8 @@ def _collect_from_paths(session: Session) -> None: reports = session.hook.pytask_collect_file_protocol( session=session, path=path, reports=session.collection_reports ) - if reports is not None: + + if reports: session.collection_reports.extend(reports) session.tasks.extend( i.node for i in reports if i.outcome == CollectionOutcome.SUCCESS @@ -85,20 +87,21 @@ def pytask_collect_file_protocol( ) -> list[CollectionReport]: """Wrap the collection of tasks from a file to collect reports.""" try: - reports = session.hook.pytask_collect_file( + new_reports = session.hook.pytask_collect_file( session=session, path=path, reports=reports ) + flat_reports = list(itertools.chain.from_iterable(new_reports)) except Exception: node = FilePathNode.from_path(path) - reports = [ + flat_reports = [ CollectionReport.from_exception( outcome=CollectionOutcome.FAIL, node=node, exc_info=sys.exc_info() ) ] - session.hook.pytask_collect_file_log(session=session, reports=reports) + session.hook.pytask_collect_file_log(session=session, reports=flat_reports) - return reports + return flat_reports @hookimpl diff --git a/src/_pytask/hookspecs.py b/src/_pytask/hookspecs.py index d830674dc..ecaee3ceb 100644 --- a/src/_pytask/hookspecs.py +++ b/src/_pytask/hookspecs.py @@ -159,7 +159,7 @@ def pytask_collect_file_protocol( """ -@hookspec(firstresult=True) +@hookspec def pytask_collect_file( session: Session, path: Path, reports: list[CollectionReport] ) -> list[CollectionReport] | None: diff --git a/src/_pytask/mark/structures.py b/src/_pytask/mark/structures.py index d044e776a..8c1e94dce 100644 --- a/src/_pytask/mark/structures.py +++ b/src/_pytask/mark/structures.py @@ -184,7 +184,7 @@ class MarkGenerator: markers: set[str] = set() """Set[str]: The set of markers.""" - def __getattr__(self, name: str) -> MarkDecorator: + def __getattr__(self, name: str) -> MarkDecorator | Any: if name[0] == "_": raise AttributeError("Marker name must NOT start with underscore") @@ -210,6 +210,11 @@ def __getattr__(self, name: str) -> MarkDecorator: stacklevel=2, ) + if name == "task": + from _pytask.task_utils import task + + return task + return MarkDecorator(Mark(name, (), {})) diff --git a/src/_pytask/models.py b/src/_pytask/models.py index abb502335..7188ae1bd 100644 --- a/src/_pytask/models.py +++ b/src/_pytask/models.py @@ -4,7 +4,9 @@ from typing import Any from typing import Dict from typing import List +from typing import Optional from typing import TYPE_CHECKING +from typing import Union import attr @@ -16,7 +18,11 @@ class CollectionMetadata: """A class for carrying metadata from functions to tasks.""" + id_ = attr.ib(default=None, type=Optional[str]) + """The id for a single parametrization.""" kwargs = attr.ib(factory=dict, type=Dict[str, Any]) """Contains kwargs which are necessary for the task function on execution.""" markers = attr.ib(factory=list, type=List["Mark"]) """Contains the markers of the function.""" + name = attr.ib(default=None, type=Union[str, None]) + """The name of the task function.""" diff --git a/src/_pytask/parametrize.py b/src/_pytask/parametrize.py index 73e170865..5c3daf5c1 100644 --- a/src/_pytask/parametrize.py +++ b/src/_pytask/parametrize.py @@ -15,11 +15,10 @@ from _pytask.console import TASK_ICON from _pytask.mark import Mark from _pytask.mark import MARK_GEN as mark # noqa: N811 -from _pytask.mark_utils import has_marker from _pytask.mark_utils import remove_markers_from_func from _pytask.nodes import find_duplicates +from _pytask.parametrize_utils import arg_value_to_id_component from _pytask.session import Session -from _pytask.task_utils import parse_task_marker def parametrize( @@ -104,11 +103,6 @@ def pytask_parametrize_task( "en/stable/how_to_guides/bp_parametrizations.html." ) - if has_marker(obj, "task"): - parsed_name = parse_task_marker(obj) - if parsed_name is not None: - name = parsed_name - base_arg_names, arg_names, arg_values = _parse_parametrize_markers( markers, name ) @@ -350,7 +344,7 @@ def _create_parametrize_ids_components( parsed_ids = [] for i, _arg_values in enumerate(arg_values): id_components = tuple( - _arg_value_to_id_component(arg_names[j], arg_value, i, ids) + arg_value_to_id_component(arg_names[j], arg_value, i, ids) for j, arg_value in enumerate(_arg_values) ) parsed_ids.append(id_components) @@ -358,45 +352,6 @@ def _create_parametrize_ids_components( return parsed_ids -def _arg_value_to_id_component( - arg_name: str, arg_value: Any, i: int, id_func: Callable[..., Any] | None -) -> str: - """Create id component from the name and value of the argument. - - First, transform the value of the argument with a user-defined function if given. - Otherwise, take the original value. Then, if the value is a :obj:`bool`, - :obj:`float`, :obj:`int`, or :obj:`str`, cast it to a string. Otherwise, define a - placeholder value from the name of the argument and the iteration. - - Parameters - ---------- - arg_name : str - Name of the parametrized function argument. - arg_value : Any - Value of the argument. - i : int - The ith iteration of the parametrization. - id_func : Union[Callable[..., Any], None] - A callable which maps argument values to :obj:`bool`, :obj:`float`, :obj:`int`, - or :obj:`str` or anything else. Any object with a different dtype than the first - will be mapped to an auto-generated id component. - - Returns - ------- - id_component : str - A part of the final parametrized id. - - """ - id_component = id_func(arg_value) if id_func is not None else None - if isinstance(id_component, (bool, float, int, str)): - id_component = str(id_component) - elif isinstance(arg_value, (bool, float, int, str)): - id_component = str(arg_value) - else: - id_component = arg_name + str(i) - return id_component - - @hookimpl def pytask_parametrize_kwarg_to_marker(obj: Any, kwargs: dict[str, str]) -> None: """Add some parametrized keyword arguments as decorator.""" diff --git a/src/_pytask/parametrize_utils.py b/src/_pytask/parametrize_utils.py new file mode 100644 index 000000000..f259c7b31 --- /dev/null +++ b/src/_pytask/parametrize_utils.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Any +from typing import Callable + + +def arg_value_to_id_component( + arg_name: str, arg_value: Any, i: int, id_func: Callable[..., Any] | None +) -> str: + """Create id component from the name and value of the argument. + + First, transform the value of the argument with a user-defined function if given. + Otherwise, take the original value. Then, if the value is a :obj:`bool`, + :obj:`float`, :obj:`int`, or :obj:`str`, cast it to a string. Otherwise, define a + placeholder value from the name of the argument and the iteration. + + Parameters + ---------- + arg_name : str + Name of the parametrized function argument. + arg_value : Any + Value of the argument. + i : int + The ith iteration of the parametrization. + id_func : Union[Callable[..., Any], None] + A callable which maps argument values to :obj:`bool`, :obj:`float`, :obj:`int`, + or :obj:`str` or anything else. Any object with a different dtype than the first + will be mapped to an auto-generated id component. + + Returns + ------- + id_component : str + A part of the final parametrized id. + + """ + id_component = id_func(arg_value) if id_func is not None else None + if isinstance(id_component, (bool, float, int, str)): + id_component = str(id_component) + elif isinstance(arg_value, (bool, float, int, str)): + id_component = str(arg_value) + else: + id_component = arg_name + str(i) + return id_component diff --git a/src/_pytask/task.py b/src/_pytask/task.py index 67dcf628f..041d4b0f5 100644 --- a/src/_pytask/task.py +++ b/src/_pytask/task.py @@ -7,8 +7,10 @@ from _pytask.config import hookimpl from _pytask.mark_utils import has_marker from _pytask.nodes import PythonFunctionTask +from _pytask.report import CollectionReport from _pytask.session import Session -from _pytask.task_utils import parse_task_marker +from _pytask.task_utils import COLLECTED_TASKS +from _pytask.task_utils import parse_collected_tasks_with_task_marker @hookimpl @@ -17,6 +19,45 @@ def pytask_parse_config(config: dict[str, Any]) -> None: config["markers"]["task"] = "Mark a function as a task regardless of its name." +@hookimpl(trylast=True) +def pytask_collect_file( + session: Session, path: Path, reports: list[CollectionReport] +) -> list[CollectionReport] | None: + """Collect a file.""" + if ( + any(path.match(pattern) for pattern in session.config["task_files"]) + and COLLECTED_TASKS[path] + ): + tasks = COLLECTED_TASKS[path] + + name_to_function = parse_collected_tasks_with_task_marker(tasks) + + collected_reports = [] + for name, function in name_to_function.items(): + session.hook.pytask_parametrize_kwarg_to_marker( + obj=function, + kwargs=function.pytask_meta.kwargs, # type: ignore[attr-defined] + ) + + if has_marker(function, "parametrize"): + names_and_objects = session.hook.pytask_parametrize_task( + session=session, name=name, obj=function + ) + else: + names_and_objects = [(name, function)] + + for name_, obj_ in names_and_objects: + report = session.hook.pytask_collect_task_protocol( + session=session, reports=reports, path=path, name=name_, obj=obj_ + ) + if report is not None: + collected_reports.append(report) + + return collected_reports + else: + return None + + @hookimpl def pytask_collect_task( session: Session, path: Path, name: str, obj: Any @@ -29,9 +70,6 @@ def pytask_collect_task( """ if has_marker(obj, "task") and callable(obj): - parsed_name = parse_task_marker(obj) - if parsed_name is not None: - name = parsed_name return PythonFunctionTask.from_path_name_function_session( path, name, obj, session ) diff --git a/src/_pytask/task_utils.py b/src/_pytask/task_utils.py index 12508957c..bc9510429 100644 --- a/src/_pytask/task_utils.py +++ b/src/_pytask/task_utils.py @@ -1,29 +1,181 @@ """This module contains utilities related to the ``@pytask.mark.task`` decorator.""" from __future__ import annotations +import inspect +from collections import defaultdict +from pathlib import Path from typing import Any from typing import Callable from _pytask.mark import Mark -from _pytask.mark_utils import remove_markers_from_func +from _pytask.models import CollectionMetadata +from _pytask.nodes import find_duplicates +from _pytask.parametrize_utils import arg_value_to_id_component -def task(name: str | None = None) -> str: - """Parse inputs of the ``@pytask.mark.task`` decorator.""" - return name +COLLECTED_TASKS: dict[Path, list[Callable[..., Any]]] = defaultdict(list) +"""A container for collecting tasks. +Tasks marked by the ``@pytask.mark.task`` decorator can be generated in a loop where one +iteration overwrites the previous task. To retrieve the tasks later, use this dictionary +mapping from paths of modules to a list of tasks per module. -def parse_task_marker(obj: Callable[..., Any]) -> str: - """Parse the ``@pytask.mark.task`` decorator.""" - obj, task_markers = remove_markers_from_func(obj, "task") +""" - if len(task_markers) != 1: + +def task( + name: str | None = None, + *, + id: str | None = None, # noqa: A002 + kwargs: dict[Any, Any] | None = None, +) -> Callable[..., None]: + """Parse inputs of the ``@pytask.mark.task`` decorator. + + The decorator wraps task functions and stores it in the global variable + :obj:`COLLECTED_TASKS` to avoid garbage collection when the function definition is + overwritten in a loop. + + The function also attaches some metadata to the function like parsed kwargs and + markers. + + Parameters + ---------- + name : str | None + The name of the task. + id : str | None + An id for the task if it is part of a parametrization. + kwargs : dict[Any, Any] | None + A dictionary containing keyword arguments which are passed to the task when it + is executed. + + """ + + def wrapper(func: Callable[..., Any]) -> None: + unwrapped = inspect.unwrap(func) + path = Path(inspect.getfile(unwrapped)).absolute().resolve() + parsed_kwargs = {} if kwargs is None else kwargs + parsed_name = name if isinstance(name, str) else func.__name__ + + if hasattr(unwrapped, "pytask_meta"): + unwrapped.pytask_meta.name = parsed_name + unwrapped.pytask_meta.kwargs = parsed_kwargs + unwrapped.pytask_meta.markers.append(Mark("task", (), {})) + unwrapped.pytask_meta.id_ = id + else: + unwrapped.pytask_meta = CollectionMetadata( + name=parsed_name, + kwargs=parsed_kwargs, + markers=[Mark("task", (), {})], + id_=id, + ) + + COLLECTED_TASKS[path].append(unwrapped) + + # Returning None is necessary since the remaining function in the module will be + # collected again from the namespace, although we already collect the function + # from ``COLLECTED_TASKS``. + return None + + # In case the decorator is used without parentheses, wrap the function which is + # passed as the first argument with the default arguments. + if callable(name) and kwargs is None: + return task()(name) + else: + return wrapper + + +def parse_collected_tasks_with_task_marker( + tasks: list[Callable[..., Any]], +) -> dict[str, Callable[..., Any]]: + """Parse collected tasks with a task marker.""" + parsed_tasks = _parse_tasks_with_preliminary_names(tasks) + all_names = {i[0] for i in parsed_tasks} + duplicated_names = find_duplicates([i[0] for i in parsed_tasks]) + + collected_tasks = {} + for name in all_names: + if name in duplicated_names: + selected_tasks = [i for i in parsed_tasks if i[0] == name] + names_to_functions = _generate_ids_for_tasks(selected_tasks) + for unique_name, task in names_to_functions.items(): + collected_tasks[unique_name] = task + else: + collected_tasks[name] = [i[1] for i in parsed_tasks if i[0] == name][0] + + return collected_tasks + + +def _parse_tasks_with_preliminary_names( + tasks: list[Callable[..., Any]], +) -> list[tuple[str, Callable[..., Any]]]: + """Parse tasks and generate preliminary names for tasks. + + The names are preliminary since they can be duplicated and need to be extended to + properly parametrized ids. + + """ + parsed_tasks = [] + for task in tasks: + name, function = _parse_task(task) + parsed_tasks.append((name, function)) + return parsed_tasks + + +def _parse_task(task: Callable[..., Any]) -> tuple[str, Callable[..., Any]]: + """Parse a single task.""" + name = task.pytask_meta.name # type: ignore[attr-defined] + if name is None and task.__name__ == "_": raise ValueError( - "The @pytask.mark.task decorator cannot be applied more than once to a " - "single task." + "A task function either needs 'name' passed by the ``@pytask.mark.task`` " + "decorator or the function name of the task function must not be '_'." ) + else: + parsed_name = task.__name__ if name is None else name + + signature_kwargs = _parse_keyword_arguments_from_signature_defaults(task) + task.pytask_meta.kwargs = { # type: ignore[attr-defined] + **task.pytask_meta.kwargs, # type: ignore[attr-defined] + **signature_kwargs, + } + + return parsed_name, task + + +def _parse_keyword_arguments_from_signature_defaults( + task: Callable[..., Any] +) -> dict[str, Any]: + """Parse keyword arguments from signature defaults.""" + parameters = inspect.signature(task).parameters + kwargs = {} + for parameter in parameters.values(): + if parameter.default is not parameter.empty: + kwargs[parameter.name] = parameter.default + return kwargs + + +def _generate_ids_for_tasks( + tasks: list[tuple[str, Callable[..., Any]]] +) -> dict[str, Callable[..., Any]]: + """Generate unique ids for parametrized tasks.""" + parameters = inspect.signature(tasks[0][1]).parameters - task_marker = task_markers[0] - name = task(*task_marker.args, **task_marker.kwargs) - obj.pytask_meta.markers.append(Mark("task", (), {})) # type: ignore[attr-defined] - return name + out = {} + for i, (name, task) in enumerate(tasks): + if task.pytask_meta.id_ is not None: # type: ignore[attr-defined] + id_ = f"{name}[{str(task.pytask_meta.id_)}]" # type: ignore[attr-defined] + else: + stringified_args = [ + arg_value_to_id_component( + arg_name=parameter, + arg_value=task.pytask_meta.kwargs.get( # type: ignore[attr-defined] + parameter + ), + i=i, + id_func=None, + ) + for parameter in parameters + ] + id_ = "-".join(stringified_args) + id_ = f"{name}[{id_}]" + out[id_] = task + return out diff --git a/tests/test_parametrize.py b/tests/test_parametrize.py index 314669030..e783a87dc 100644 --- a/tests/test_parametrize.py +++ b/tests/test_parametrize.py @@ -8,7 +8,6 @@ import _pytask.parametrize import pytask import pytest -from _pytask.parametrize import _arg_value_to_id_component from _pytask.parametrize import _check_if_n_arg_names_matches_n_arg_values from _pytask.parametrize import _parse_arg_names from _pytask.parametrize import _parse_arg_values @@ -299,28 +298,6 @@ def task_func(): assert isinstance(session.collection_reports[0].exc_info[1], ValueError) -@pytest.mark.unit -@pytest.mark.parametrize( - "arg_name, arg_value, i, id_func, expected", - [ - ("arg", 1, 0, None, "1"), - ("arg", True, 0, None, "True"), - ("arg", False, 0, None, "False"), - ("arg", 1.0, 0, None, "1.0"), - ("arg", None, 0, None, "arg0"), - ("arg", (1,), 0, None, "arg0"), - ("arg", [1], 0, None, "arg0"), - ("arg", {1, 2}, 0, None, "arg0"), - ("arg", 1, 0, lambda x: bool(x), "True"), - ("arg", 1, 1, lambda x: None, "1"), - ("arg", [1], 2, lambda x: None, "arg2"), - ], -) -def test_arg_value_to_id_component(arg_name, arg_value, i, id_func, expected): - result = _arg_value_to_id_component(arg_name, arg_value, i, id_func) - assert result == expected - - @pytest.mark.end_to_end def test_raise_error_if_parametrization_produces_non_unique_tasks(tmp_path): tmp_path.joinpath("task_module.py").write_text( diff --git a/tests/test_parametrize_utils.py b/tests/test_parametrize_utils.py new file mode 100644 index 000000000..f0635823f --- /dev/null +++ b/tests/test_parametrize_utils.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import pytest +from _pytask.parametrize_utils import arg_value_to_id_component + + +@pytest.mark.unit +@pytest.mark.parametrize( + "arg_name, arg_value, i, id_func, expected", + [ + ("arg", 1, 0, None, "1"), + ("arg", True, 0, None, "True"), + ("arg", False, 0, None, "False"), + ("arg", 1.0, 0, None, "1.0"), + ("arg", None, 0, None, "arg0"), + ("arg", (1,), 0, None, "arg0"), + ("arg", [1], 0, None, "arg0"), + ("arg", {1, 2}, 0, None, "arg0"), + ("arg", 1, 0, lambda x: bool(x), "True"), + ("arg", 1, 1, lambda x: None, "1"), + ("arg", [1], 2, lambda x: None, "arg2"), + ], +) +def test_arg_value_to_id_component(arg_name, arg_value, i, id_func, expected): + result = arg_value_to_id_component(arg_name, arg_value, i, id_func) + assert result == expected diff --git a/tests/test_task.py b/tests/test_task.py index 316a1c227..58dda481c 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -4,6 +4,7 @@ import pytest from _pytask.nodes import create_task_name +from pytask import cli from pytask import ExitCode from pytask import main @@ -50,7 +51,8 @@ def test_task_with_task_decorator_with_parametrize(tmp_path, func_name, task_nam def {func_name}(produces): produces.write_text("Hello. It's me.") """ - tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + path_to_module = tmp_path.joinpath("task_module.py") + path_to_module.write_text(textwrap.dedent(source)) session = main({"paths": tmp_path}) @@ -58,15 +60,195 @@ def {func_name}(produces): if task_name: assert session.tasks[0].name == create_task_name( - tmp_path.joinpath("task_module.py"), f"{task_name}[out_1.txt]" + path_to_module, f"{task_name}[out_1.txt]" ) assert session.tasks[1].name == create_task_name( - tmp_path.joinpath("task_module.py"), f"{task_name}[out_2.txt]" + path_to_module, f"{task_name}[out_2.txt]" ) else: assert session.tasks[0].name == create_task_name( - tmp_path.joinpath("task_module.py"), f"{func_name}[out_1.txt]" + path_to_module, f"{func_name}[out_1.txt]" ) assert session.tasks[1].name == create_task_name( - tmp_path.joinpath("task_module.py"), f"{func_name}[out_2.txt]" + path_to_module, f"{func_name}[out_2.txt]" ) + + +@pytest.mark.end_to_end +def test_parametrization_in_for_loop(tmp_path, runner): + source = """ + import pytask + + for i in range(2): + + @pytask.mark.task + @pytask.mark.produces(f"out_{i}.txt") + def task_example(produces): + produces.write_text("Your advertisement could be here.") + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert "task_example[produces0]" in result.output + assert "task_example[produces1]" in result.output + + +@pytest.mark.end_to_end +def test_parametrization_in_for_loop_from_markers(tmp_path, runner): + source = """ + import pytask + + for i in range(2): + + @pytask.mark.task + @pytask.mark.depends_on(f"in_{i}.txt") + @pytask.mark.produces(f"out_{i}.txt") + def example(depends_on, produces): + produces.write_text(depends_on.read_text()) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("in_0.txt").write_text("Your advertisement could be here.") + tmp_path.joinpath("in_1.txt").write_text("Or here.") + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert "example[depends_on0-produces0]" in result.output + assert "example[depends_on1-produces1]" in result.output + + +@pytest.mark.end_to_end +def test_parametrization_in_for_loop_from_signature(tmp_path, runner): + source = """ + import pytask + + for i in range(2): + + @pytask.mark.task + def example(depends_on=f"in_{i}.txt", produces=f"out_{i}.txt"): + produces.write_text(depends_on.read_text()) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("in_0.txt").write_text("Your advertisement could be here.") + tmp_path.joinpath("in_1.txt").write_text("Or here.") + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert "example[in_0.txt-out_0.txt]" in result.output + assert "example[in_1.txt-out_1.txt]" in result.output + + +@pytest.mark.end_to_end +def test_parametrization_in_for_loop_from_markers_and_args(tmp_path, runner): + source = """ + import pytask + + for i in range(2): + + @pytask.mark.task + @pytask.mark.produces(f"out_{i}.txt") + def example(produces, i=i): + produces.write_text(str(i)) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert "example[produces0-0]" in result.output + assert "example[produces1-1]" in result.output + + +@pytest.mark.end_to_end +def test_parametrization_in_for_loop_from_decorator(tmp_path, runner): + source = """ + import pytask + + for i in range(2): + + @pytask.mark.task(name="deco_task", kwargs={"i": i, "produces": f"out_{i}.txt"}) + def example(produces, i): + produces.write_text(str(i)) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert "deco_task[out_0.txt-0]" in result.output + assert "deco_task[out_1.txt-1]" in result.output + + +@pytest.mark.end_to_end +def test_parametrization_in_for_loop_with_ids(tmp_path, runner): + source = """ + import pytask + + for i in range(2): + + @pytask.mark.task( + "deco_task", id=i, kwargs={"i": i, "produces": f"out_{i}.txt"} + ) + def example(produces, i): + produces.write_text(str(i)) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert "deco_task[0]" in result.output + assert "deco_task[1]" in result.output + + +@pytest.mark.end_to_end +def test_parametrization_in_for_loop_with_error(tmp_path, runner): + source = """ + import pytask + + for i in range(2): + + @pytask.mark.task + def task_example(produces=f"out_{i}.txt"): + raise ValueError + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.FAILED + assert "2 Failed" in result.output + assert "Traceback" in result.output + assert "task_example[out_0.txt]" in result.output + assert "task_example[out_1.txt]" in result.output + + +@pytest.mark.end_to_end +def test_parametrization_in_for_loop_from_decorator_w_irregular_dicts(tmp_path, runner): + source = """ + import pytask + + ID_TO_KWARGS = { + "first": {"i": 0, "produces": "out_0.txt"}, + "second": {"produces": "out_1.txt"}, + } + + for id_, kwargs in ID_TO_KWARGS.items(): + + @pytask.mark.task(name="deco_task", id=id_, kwargs=kwargs) + def example(produces, i): + produces.write_text(str(i)) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.FAILED + assert "deco_task[first]" in result.output + assert "deco_task[second]" in result.output + assert "1 Succeeded" + assert "1 Failed" + assert "TypeError: example() missing 1 required" in result.output