diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..fd5df4b --- /dev/null +++ b/.coveragerc @@ -0,0 +1,5 @@ +[report] +exclude_lines = + pragma: no cover + if TYPE_CHECKING.*: + \.\.\. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1817a2f..b627f37 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -6,4 +6,4 @@ Provide a description and/or bullet points to describe the changes in this PR. - [ ] Reference issues which can be closed due to this PR with "Closes #x". - [ ] Review whether the documentation needs to be updated. -- [ ] Document PR in docs/changes.rst. +- [ ] Document PR in CHANGES.md. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 882b9bb..56727f5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,15 +48,6 @@ jobs: shell: bash -l {0} run: bash <(curl -s https://codecov.io/bash) -F unit -c - # - name: Run integration tests. - # shell: bash -l {0} - # run: tox -e pytest -- -m integration --cov=./ --cov-report=xml -n auto - - # - name: Upload coverage reports of integration tests. - # if: runner.os == 'Linux' && matrix.python-version == '3.8' - # shell: bash -l {0} - # run: bash <(curl -s https://codecov.io/bash) -F integration -c - - name: Run end-to-end tests. shell: bash -l {0} run: tox -e pytest -- -m end_to_end --cov=./ --cov-report=xml -n auto diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11de5e6..c3d6d49 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,9 +23,6 @@ repos: - id: python-no-eval - id: python-no-log-warn - id: python-use-type-annotations - - id: rst-backticks - - id: rst-directive-colons - - id: rst-inline-touching-normal - id: text-unicode-replacement-char - repo: https://github.com/asottile/pyupgrade rev: v2.32.0 @@ -45,11 +42,6 @@ repos: rev: 22.3.0 hooks: - id: black -- repo: https://github.com/asottile/blacken-docs - rev: v1.12.1 - hooks: - - id: blacken-docs - additional_dependencies: [black] - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: @@ -71,10 +63,6 @@ repos: pydocstyle, Pygments, ] -- repo: https://github.com/PyCQA/doc8 - rev: 0.11.1 - hooks: - - id: doc8 - repo: https://github.com/econchick/interrogate rev: 1.5.0 hooks: @@ -84,11 +72,12 @@ repos: rev: v2.1.0 hooks: - id: codespell - args: [-L unparseable] - repo: https://github.com/mgedmin/check-manifest rev: "0.48" hooks: - id: check-manifest + args: [--no-build-isolation] + additional_dependencies: [setuptools-scm, toml] - repo: meta hooks: - id: check-hooks-apply diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..cb54a22 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,58 @@ +# Changes + +This is a record of all past pytask-stata releases and what went into +them in reverse chronological order. Releases follow [semantic +versioning](https://semver.org/) and all releases are available on +[PyPI](https://pypi.org/project/pytask-stata) and +[Anaconda.org](https://anaconda.org/conda-forge/pytask-stata). + +## 0.2.0 - 2022-xx-xx + +- {pull}`20` removes an unnecessary hook implementation. + +## 0.1.2 - 2022-02-08 + +- {pull}`19` fixes the minimum python and + pytask version. + +## 0.1.1 - 2022-02-07 + +- {pull}`16` skips concurrent CI builds. +- {pull}`17` deprecates Python 3.6 and adds + support for Python 3.10. + +## 0.1.0 - 2021-07-21 + +- {pull}`11` fixes the `README.rst`. +- {pull}`13` replaces versioneer with + setuptools-scm. +- {pull}`14` fixes tests and aligns + pytask-stata with pytask v0.1.0. + +## 0.0.6 - 2021-03-05 + +- {pull}`10` fixes the version of the package. + +## 0.0.5 - 2021-03-04 + +- {pull}`7` fix some post-release issues. +- {pull}`9` adds dependencies to `setup.py`. + +## 0.0.4 - 2021-02-25 + +- {pull}`6` prepares pytask-stata to be + published on PyPI, adds versioneer and more. + +## 0.0.3 - 2021-01-16 + +- {pull}`4` removes log file handling on UNIX + and raises an error if run in parallel. + +## 0.0.2 - 2020-10-30 + +- {pull}`1` makes pytask-stata work with pytask + v0.0.9. + +## 0.0.1 - 2020-10-04 + +- Release v0.0.1. diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index b8bff6b..0000000 --- a/CHANGES.rst +++ /dev/null @@ -1,70 +0,0 @@ -Changes -======= - -This is a record of all past pytask-stata releases and what went into them in reverse -chronological order. Releases follow `semantic versioning `_ and -all releases are available on `PyPI `_ and -`Anaconda.org `_. - - -0.2.0 - 2022-xx-xx ------------------- - -- :gh`:`20` removes an unnecessary hook implementation. - - -0.1.2 - 2022-02-08 ------------------- - -- :gh:`19` fixes the minimum python and pytask version. - - -0.1.1 - 2022-02-07 ------------------- - -- :gh:`16` skips concurrent CI builds. -- :gh:`17` deprecates Python 3.6 and adds support for Python 3.10. - - -0.1.0 - 2021-07-21 ------------------- - -- :gh:`11` fixes the ``README.rst``. -- :gh:`13` replaces versioneer with setuptools-scm. -- :gh:`14` fixes tests and aligns pytask-stata with pytask v0.1.0. - - -0.0.6 - 2021-03-05 ------------------- - -- :gh:`10` fixes the version of the package. - - -0.0.5 - 2021-03-04 ------------------- - -- :gh:`7` fix some post-release issues. -- :gh:`9` adds dependencies to ``setup.py``. - - -0.0.4 - 2021-02-25 ------------------- - -- :gh:`6` prepares pytask-stata to be published on PyPI, adds versioneer and more. - - -0.0.3 - 2021-01-16 ------------------- - -- :gh:`4` removes log file handling on UNIX and raises an error if run in parallel. - -0.0.2 - 2020-10-30 ------------------- - -- :gh:`1` makes pytask-stata work with pytask v0.0.9. - - -0.0.1 - 2020-10-04 ------------------- - -- Release v0.0.1. diff --git a/LICENSE b/LICENSE index 4c96cf3..f4c44ec 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2020-2021 Tobias Raabe +Copyright 2020 Tobias Raabe Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software diff --git a/MANIFEST.in b/MANIFEST.in index a7c8a20..6782aaf 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,12 +1,10 @@ -prune .conda prune tests -exclude *.rst +exclude .coveragerc +exclude *.md exclude *.yml exclude *.yaml exclude tox.ini -include README.rst +include README.md include LICENSE -include versioneer.py -include src/pytask_stata/_version.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..affe05d --- /dev/null +++ b/README.md @@ -0,0 +1,165 @@ +# pytask-stata + +[![PyPI](https://img.shields.io/pypi/v/pytask-stata?color=blue)](https://pypi.org/project/pytask-stata) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pytask-stata)](https://pypi.org/project/pytask-stata) +[![image](https://img.shields.io/conda/vn/conda-forge/pytask-stata.svg)](https://anaconda.org/conda-forge/pytask-stata) +[![image](https://img.shields.io/conda/pn/conda-forge/pytask-stata.svg)](https://anaconda.org/conda-forge/pytask-stata) +[![PyPI - License](https://img.shields.io/pypi/l/pytask-stata)](https://pypi.org/project/pytask-stata) +[![image](https://img.shields.io/github/workflow/status/pytask-dev/pytask-stata/main/main)](https://github.com/pytask-dev/pytask-stata/actions?query=branch%3Amain) +[![image](https://codecov.io/gh/pytask-dev/pytask-stata/branch/main/graph/badge.svg)](https://codecov.io/gh/pytask-dev/pytask-stata) +[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pytask-dev/pytask-stata/main.svg)](https://results.pre-commit.ci/latest/github/pytask-dev/pytask-stata/main) +[![image](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + +------------------------------------------------------------------------ + +Run Stata\'s do-files with pytask. + +## Installation + +pytask-stata is available on +[PyPI](https://pypi.org/project/pytask-stata) and +[Anaconda.org](https://anaconda.org/conda-forge/pytask-stata). Install +it with + +``` console +$ pip install pytask-stata + +# or + +$ conda install -c conda-forge pytask-stata +``` + +You also need to have Stata installed on your system and have the +executable on your system\'s PATH. If you do not know how to do it, +[here](https://superuser.com/a/284351) is an explanation. + +## Usage + +Similarly to normal task functions which execute Python code, you define +tasks to execute scripts written in Stata with Python functions. The +difference is that the function body does not contain any logic, but the +decorator tells pytask how to handle the task. + +Here is an example where you want to run `script.do`. + +``` python +import pytask + + +@pytask.mark.stata(script="script.do") +@pytask.mark.produces("auto.dta") +def task_run_do_file(): + pass +``` + +When executing a do-file, the current working directory changes to the +directory where the script is located. This allows you, for example, to +reference every data set you want to read with a relative path from the +script. + +### Dependencies and Products + +Dependencies and products can be added as with a normal pytask task +using the `@pytask.mark.depends_on` and `@pytask.mark.produces` +decorators. which is explained in this +[tutorial](https://pytask-dev.readthedocs.io/en/stable/tutorials/defining_dependencies_products.html). + +### Accessing dependencies and products in the script + +The decorator can be used to pass command line arguments to your Stata +executable. For example, pass the path of the product with + +``` python +@pytask.mark.stata(script="script.do", options="auto.dta") +@pytask.mark.produces("auto.dta") +def task_run_do_file(): + pass +``` + +And in your `script.do`, you can intercept the value with + +``` do +* Intercept command line argument and save to macro named 'produces'. +args produces + +sysuse auto, clear +save "`produces'" +``` + +The relative path inside the do-file works only because the pytask-stata +switches the current working directory to the directory of the do-file +before the task is executed. + +To make the task independent from the current working directory, pass +the full path as an command line argument. Here is an example. + +``` python +# Absolute path to the build directory. +from src.config import BLD + + +@pytask.mark.stata(script="script.do", options=BLD / "auto.dta") +@pytask.mark.produces(BLD / "auto.dta") +def task_run_do_file(): + pass +``` + +### Repeating tasks with different scripts or inputs + +You can also parametrize the execution of scripts, meaning executing +multiple do-files as well as passing different command line arguments to +the same do-file. + +The following task executes two do-files which produce different +outputs. + +``` python +for i in range(2): + + @pytask.mark.task + @pytask.mark.stata(script=f"script_{i}.do", options=f"{i}.dta") + @pytask.mark.produces(f"{i}.dta") + def task_execute_do_file(): + pass +``` + +## Configuration + +pytask-stata can be configured with the following options. + +*`stata_keep_log`* + +Use this option to keep the `.log` files which are produced for +every task. This option is useful to debug Stata tasks. Set the +option via the configuration file with + +```toml +[tool.pytask.ini_options] +stata_keep_log = true +``` + +The option is also available in the command line interface via the +`--stata-keep-log` flag. + + +*`stata_check_log_lines`* + +Use this option to vary the number of lines in the log file which +are checked for error codes. It also controls the number of lines +displayed on errors. Use any integer greater than zero. Here is the +entry in the configuration file + +```toml +[tool.pytask.ini_options] +stata_check_log_lines = 10 +``` + +and here via the command line interface + +``` console +$ pytask build --stata-check-log-lines 10 +``` + +## Changes + +Consult the [release notes](CHANGES.md) to find out about what is new. diff --git a/README.rst b/README.rst deleted file mode 100644 index 2d18586..0000000 --- a/README.rst +++ /dev/null @@ -1,247 +0,0 @@ -.. image:: https://img.shields.io/pypi/v/pytask-stata?color=blue - :alt: PyPI - :target: https://pypi.org/project/pytask-stata - -.. image:: https://img.shields.io/pypi/pyversions/pytask-stata - :alt: PyPI - Python Version - :target: https://pypi.org/project/pytask-stata - -.. image:: https://img.shields.io/conda/vn/conda-forge/pytask-stata.svg - :target: https://anaconda.org/conda-forge/pytask-stata - -.. image:: https://img.shields.io/conda/pn/conda-forge/pytask-stata.svg - :target: https://anaconda.org/conda-forge/pytask-stata - -.. image:: https://img.shields.io/pypi/l/pytask-stata - :alt: PyPI - License - :target: https://pypi.org/project/pytask-stata - -.. image:: https://img.shields.io/github/workflow/status/pytask-dev/pytask-stata/main/main - :target: https://github.com/pytask-dev/pytask-stata/actions?query=branch%3Amain - -.. image:: https://codecov.io/gh/pytask-dev/pytask-stata/branch/main/graph/badge.svg - :target: https://codecov.io/gh/pytask-dev/pytask-stata - -.. image:: https://results.pre-commit.ci/badge/github/pytask-dev/pytask-stata/main.svg - :target: https://results.pre-commit.ci/latest/github/pytask-dev/pytask-stata/main - :alt: pre-commit.ci status - -.. image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black - ------- - -pytask-stata -============ - -Run Stata's do-files with pytask. - - -Installation ------------- - -pytask-stata is available on `PyPI `_ and -`Anaconda.org `_. Install it with - -.. code-block:: console - - $ pip install pytask-stata - - # or - - $ conda install -c conda-forge pytask-stata - -You also need to have Stata installed on your system and have the executable on your -system's PATH. If you do not know how to do it, `here `_ -is an explanation. - - -Usage ------ - -Similarly to normal task functions which execute Python code, you define tasks to -execute scripts written in Stata with Python functions. The difference is that the -function body does not contain any logic, but the decorator tells pytask how to handle -the task. - -Here is an example where you want to run ``script.do``. - -.. code-block:: python - - import pytask - - - @pytask.mark.stata - @pytask.mark.depends_on("script.do") - @pytask.mark.produces("auto.dta") - def task_run_do_file(): - pass - -When executing a do-file, the current working directory changes to the directory of the -script which is executed. - - -Multiple dependencies and products -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -What happens if a task has more dependencies? Using a list, the do-file which should be -executed must be found in the first position of the list. - -.. code-block:: python - - @pytask.mark.stata - @pytask.mark.depends_on(["script.do", "input.dta"]) - @pytask.mark.produces("output.dta") - def task_run_do_file(): - pass - -If you use a dictionary to pass dependencies to the task, pytask-stata will, first, look -for a ``"source"`` key in the dictionary and, secondly, under the key ``0``. - -.. code-block:: python - - @pytask.mark.depends_on({"source": "script.do", "input": "input.dta"}) - def task_run_do_file(): - pass - - - # or - - - @pytask.mark.depends_on({0: "script.do", "input": "input.dta"}) - def task_run_do_file(): - pass - - - # or two decorators for the function, if you do not assign a name to the input. - - - @pytask.mark.depends_on({"source": "script.do"}) - @pytask.mark.depends_on("input.dta") - def task_run_do_file(): - pass - - - -Command Line Arguments -~~~~~~~~~~~~~~~~~~~~~~ - -The decorator can be used to pass command line arguments to your Stata executable. For -example, pass the path of the product with - -.. code-block:: python - - @pytask.mark.stata("auto.dta") - @pytask.mark.depends_on("script.do") - @pytask.mark.produces("auto.dta") - def task_run_do_file(): - pass - -And in your ``script.do``, you can intercept the value with - -.. code-block:: do - - * Intercept command line argument and save to macro named 'produces'. - args produces - - sysuse auto, clear - save "`produces'" - -The relative path inside the do-file works only because the pytask-stata switches the -current working directory to the directory of the do-file before the task is executed. -This is necessary precaution. - -To make the task independent from the current working directory, pass the full path as -an command line argument. Here is an example. - -.. code-block:: python - - # Absolute path to the build directory. - from src.config import BLD - - - @pytask.mark.stata(BLD / "auto.dta") - @pytask.mark.depends_on("script.do") - @pytask.mark.produces(BLD / "auto.dta") - def task_run_do_file(): - pass - - -Parametrization -~~~~~~~~~~~~~~~ - -You can also parametrize the execution of scripts, meaning executing multiple do-files -as well as passing different command line arguments to the same do-file. - -The following task executes two do-files which produce different outputs. - -.. code-block:: python - - @pytask.mark.stata - @pytask.mark.parametrize( - "depends_on, produces", [("script_1.do", "1.dta"), ("script_2.do", "2.dta")] - ) - def task_execute_do_file(): - pass - - -If you want to pass different command line arguments to the same do-file, you have to -include the ``@pytask.mark.stata`` decorator in the parametrization just like with -``@pytask.mark.depends_on`` and ``@pytask.mark.produces``. - -.. code-block:: python - - @pytask.mark.depends_on("script.do") - @pytask.mark.parametrize( - "produces, stata", - [("output_1.dta", ("1",)), ("output_2.dta", ("2",))], - ) - def task_execute_do_file(): - pass - - -Configuration -------------- - -pytask-stata can be configured with the following options. - -stata_keep_log - Use this option to keep the ``.log`` files which are produced for every task. This - option is useful to debug Stata tasks. Set the option via the configuration file - with - - .. code-block:: ini - - stata_keep_log = (True|true|1|False|false|0) - - The option is also available in the command line interface via the - ``--stata-keep-log`` flag. - -stata_check_log_lines - Use this option to vary the number of lines in the log file which are checked for - error codes. It also controls the number of lines displayed on errors. Use any - integer greater than zero. Here is the entry in the configuration file - - .. code-block:: ini - - stata_check_log_lines = 10 - - and here via the command line interface - - .. code-block:: console - - $ pytask build --stata-check-log-lines 10 - -stata_source_key - If you want to change the name of the key which identifies the do file, change the - following default configuration in your pytask configuration file. - - .. code-block:: ini - - stata_source_key = source - - -Changes -------- - -Consult the `release notes `_ to find out about what is new. diff --git a/environment.yml b/environment.yml index b3911fb..4605ad1 100644 --- a/environment.yml +++ b/environment.yml @@ -5,25 +5,17 @@ channels: - nodefaults dependencies: - - python >=3.6 + - python >3.7 - pip - setuptools_scm - toml - # Conda - - anaconda-client - - conda-build - - conda-verify - # Package dependencies - - pytask >=0.1.0 - - pytask-parallel >=0.0.9 + - pytask >=0.2 + - pytask-parallel >=0.2 # Misc - black - - bumpversion - - jupyterlab - - pdbpp - pre-commit - pytest-cov - pytest-xdist diff --git a/pyproject.toml b/pyproject.toml index 8823673..692c35b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,6 @@ [build-system] requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.0"] +build-backend = "setuptools.build_meta" [tool.setuptools_scm] diff --git a/setup.cfg b/setup.cfg index d42a5b7..f8932a9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,8 @@ [metadata] name = pytask_stata description = Execute do-files with Stata and pytask. -long_description = file: README.rst -long_description_content_type = text/x-rst +long_description = file: README.md +long_description_content_type = text/markdown url = https://github.com/pytask-dev/pytask-stata author = Tobias Raabe author_email = raabe@posteo.de @@ -10,7 +10,7 @@ license = MIT license_file = LICENSE platforms = any classifiers = - Development Status :: 3 - Alpha + Development Status :: 4 - Beta License :: OSI Approved :: MIT License Operating System :: OS Independent Programming Language :: Python :: 3 @@ -23,13 +23,13 @@ project_urls = Documentation = https://github.com/pytask-dev/pytask-stata Github = https://github.com/pytask-dev/pytask-stata Tracker = https://github.com/pytask-dev/pytask-stata/issues - Changelog = https://github.com/pytask-dev/pytask-stata/blob/main/CHANGES.rst + Changelog = https://github.com/pytask-dev/pytask-stata/blob/main/CHANGES.md [options] packages = find: install_requires = click - pytask>=0.1.7 + pytask>=0.2 python_requires = >=3.7 include_package_data = True package_dir = =src diff --git a/setup.py b/setup.py deleted file mode 100644 index c21a9ee..0000000 --- a/setup.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import annotations - -from setuptools import setup - - -if __name__ == "__main__": - setup() diff --git a/src/pytask_stata/cli.py b/src/pytask_stata/cli.py index 5ceda23..3695031 100644 --- a/src/pytask_stata/cli.py +++ b/src/pytask_stata/cli.py @@ -2,11 +2,11 @@ from __future__ import annotations import click -from _pytask.config import hookimpl +from pytask import hookimpl @hookimpl -def pytask_extend_command_line_interface(cli): +def pytask_extend_command_line_interface(cli: click.Group) -> None: """Add stata related options to the command line interface.""" additional_parameters = [ click.Option( diff --git a/src/pytask_stata/collect.py b/src/pytask_stata/collect.py index eaa96be..fa535da 100644 --- a/src/pytask_stata/collect.py +++ b/src/pytask_stata/collect.py @@ -1,120 +1,135 @@ """Collect tasks.""" from __future__ import annotations -import copy import functools import subprocess -from typing import Iterable -from typing import Sequence - -from _pytask.config import hookimpl -from _pytask.mark_utils import get_specific_markers_from_task -from _pytask.nodes import FilePathNode -from _pytask.parametrize import _copy_func +from pathlib import Path +from types import FunctionType + +from pytask import depends_on +from pytask import has_mark +from pytask import hookimpl +from pytask import Mark +from pytask import parse_nodes +from pytask import produces +from pytask import remove_marks +from pytask import Task from pytask_stata.shared import convert_task_id_to_name_of_log_file -from pytask_stata.shared import get_node_from_dictionary - - -def stata(options: str | Iterable[str] | None = None): - """Specify command line options for Stata. - - Parameters - ---------- - options : Optional[Union[str, Iterable[str]]] - One or multiple command line options passed to Stata. - - """ - options = _to_list(options) if options is not None else [] - options = [str(i) for i in options] - return options +from pytask_stata.shared import stata -def run_stata_script(stata, cwd): +def run_stata_script( + executable: str, script: Path, options: list[str], log_name: list[str], cwd: Path +) -> None: """Run an R script.""" - print("Executing " + " ".join(stata) + ".") # noqa: T001 - subprocess.run(stata, cwd=cwd, check=True) + cmd = [executable, "-e", "do", script.as_posix(), *options, *log_name] + print("Executing " + " ".join(cmd) + ".") # noqa: T001 + subprocess.run(cmd, cwd=cwd, check=True) @hookimpl -def pytask_collect_task_teardown(session, task): +def pytask_collect_task(session, path, name, obj): """Perform some checks and prepare the task function.""" - if get_specific_markers_from_task(task, "stata"): - source = get_node_from_dictionary( - task.depends_on, session.config["stata_source_key"] - ) - if not (isinstance(source, FilePathNode) and source.value.suffix == ".do"): + __tracebackhide__ = True + + if ( + (name.startswith("task_") or has_mark(obj, "task")) + and callable(obj) + and has_mark(obj, "stata") + ): + obj, marks = remove_marks(obj, "stata") + + if len(marks) > 1: raise ValueError( - "The first dependency of a Stata task must be the do-file." + f"Task {name!r} has multiple @pytask.mark.stata marks, but only one is " + "allowed." ) - stata_function = _copy_func(run_stata_script) - stata_function.pytaskmark = copy.deepcopy(task.function.pytaskmark) + mark = _parse_stata_mark(mark=marks[0]) + script, options = stata(**marks[0].kwargs) - merged_marks = _merge_all_markers(task) - args = stata(*merged_marks.args, **merged_marks.kwargs) - options = _prepare_cmd_options(session, task, args) - stata_function = functools.partial( - stata_function, stata=options, cwd=task.path.parent + obj.pytask_meta.markers.append(mark) + + dependencies = parse_nodes(session, path, name, obj, depends_on) + products = parse_nodes(session, path, name, obj, produces) + + markers = obj.pytask_meta.markers if hasattr(obj, "pytask_meta") else [] + kwargs = obj.pytask_meta.kwargs if hasattr(obj, "pytask_meta") else {} + + task = Task( + base_name=name, + path=path, + function=_copy_func(run_stata_script), + depends_on=dependencies, + produces=products, + markers=markers, + kwargs=kwargs, ) - task.function = stata_function + script_node = session.hook.pytask_collect_node( + session=session, path=path, node=script + ) + if isinstance(task.depends_on, dict): + task.depends_on["__script"] = script_node + else: + task.depends_on = {0: task.depends_on, "__script": script_node} -def _merge_all_markers(task): - """Combine all information from markers for the Stata function.""" - stata_marks = get_specific_markers_from_task(task, "stata") - mark = stata_marks[0] - for mark_ in stata_marks[1:]: - mark = mark.combined_with(mark_) - return mark + if session.config["platform"] == "win32": + log_name = convert_task_id_to_name_of_log_file(task.short_name) + log_name_arg = [f"-{log_name}"] + else: + log_name_arg = [] + + stata_function = functools.partial( + task.function, + executable=session.config["stata"], + script=task.depends_on["__script"].path, + options=options, + log_name=log_name_arg, + cwd=task.path.parent, + ) + task.function = stata_function -def _prepare_cmd_options(session, task, args): - """Prepare the command line arguments to execute the do-file. + return task - The last entry changes the name of the log file. We take the task id as a name which - is unique and does not cause any errors when parallelizing the execution. - """ - source = get_node_from_dictionary( - task.depends_on, session.config["stata_source_key"] - ) +def _parse_stata_mark(mark): + """Parse a Julia mark.""" + script, options = stata(**mark.kwargs) - cmd_options = [ - session.config["stata"], - "-e", - "do", - source.path.as_posix(), - *args, - ] - if session.config["platform"] == "win32": - log_name = convert_task_id_to_name_of_log_file(task.name) - cmd_options.append(f"-{log_name}") + parsed_kwargs = {} + for arg_name, value, default in [ + ("script", script, None), + ("options", options, []), + ]: + parsed_kwargs[arg_name] = value if value else default - return cmd_options + mark = Mark("stata", (), parsed_kwargs) + return mark -def _to_list(scalar_or_iter): - """Convert scalars and iterables to list. +def _copy_func(func: FunctionType) -> FunctionType: + """Create a copy of a function. - Parameters - ---------- - scalar_or_iter : str or list + Based on https://stackoverflow.com/a/13503277/7523785. - Returns + Example ------- - list - - Examples - -------- - >>> _to_list("a") - ['a'] - >>> _to_list(["b"]) - ['b'] + >>> def _func(): pass + >>> copied_func = _copy_func(_func) + >>> _func is copied_func + False """ - return ( - [scalar_or_iter] - if isinstance(scalar_or_iter, str) or not isinstance(scalar_or_iter, Sequence) - else list(scalar_or_iter) + new_func = FunctionType( + func.__code__, + func.__globals__, + name=func.__name__, + argdefs=func.__defaults__, + closure=func.__closure__, ) + new_func = functools.update_wrapper(new_func, func) + new_func.__kwdefaults__ = func.__kwdefaults__ + return new_func diff --git a/src/pytask_stata/config.py b/src/pytask_stata/config.py index cbc5935..a37ed2d 100644 --- a/src/pytask_stata/config.py +++ b/src/pytask_stata/config.py @@ -3,10 +3,10 @@ import shutil import sys +from typing import Any +from typing import Callable -from _pytask.config import hookimpl -from _pytask.shared import convert_truthy_or_falsy_to_bool -from _pytask.shared import get_first_non_none_value +from pytask import hookimpl from pytask_stata.shared import STATA_COMMANDS @@ -24,15 +24,15 @@ def pytask_parse_config(config, config_from_cli, config_from_file): None, ) - config["stata_keep_log"] = get_first_non_none_value( + config["stata_keep_log"] = _get_first_non_none_value( config_from_cli, config_from_file, key="stata_keep_log", - callback=convert_truthy_or_falsy_to_bool, + callback=_convert_truthy_or_falsy_to_bool, default=False, ) - config["stata_check_log_lines"] = get_first_non_none_value( + config["stata_check_log_lines"] = _get_first_non_none_value( config_from_cli, config_from_file, key="stata_check_log_lines", @@ -40,8 +40,6 @@ def pytask_parse_config(config, config_from_cli, config_from_file): default=10, ) - config["stata_source_key"] = config_from_file.get("stata_source_key", "source") - def _nonnegative_nonzero_integer(x): """Check for nonnegative and nonzero integer.""" @@ -58,3 +56,46 @@ def _nonnegative_nonzero_integer(x): raise ValueError("'stata_check_log_lines' must be greater than zero.") return x + + +def _get_first_non_none_value( + *configs: dict[str, Any], + key: str, + default: Any | None = None, + callback: Callable[..., Any] | None = None, +) -> Any: + """Get the first non-None value for a key from a list of dictionaries. + + This function allows to prioritize information from many configurations by changing + the order of the inputs while also providing a default. + + Examples + -------- + >>> _get_first_non_none_value({"a": None}, {"a": 1}, key="a") + 1 + >>> _get_first_non_none_value({"a": None}, {"a": None}, key="a", default="default") + 'default' + >>> _get_first_non_none_value({}, {}, key="a", default="default") + 'default' + >>> _get_first_non_none_value({"a": None}, {"a": "b"}, key="a") + 'b' + + """ + callback = (lambda x: x) if callback is None else callback # noqa: E731 + processed_values = (callback(config.get(key)) for config in configs) + return next((value for value in processed_values if value is not None), default) + + +def _convert_truthy_or_falsy_to_bool(x: bool | str | None) -> bool: + """Convert truthy or falsy value in .ini to Python boolean.""" + if x in [True, "True", "true", "1"]: + out = True + elif x in [False, "False", "false", "0"]: + out = False + elif x in [None, "None", "none"]: + out = None + else: + raise ValueError( + f"Input {x!r} is neither truthy (True, true, 1) or falsy (False, false, 0)." + ) + return out diff --git a/src/pytask_stata/execute.py b/src/pytask_stata/execute.py index cf7ebb7..1879bda 100644 --- a/src/pytask_stata/execute.py +++ b/src/pytask_stata/execute.py @@ -3,20 +3,16 @@ import re -from _pytask.config import hookimpl -from _pytask.mark_utils import get_specific_markers_from_task +from pytask import has_mark +from pytask import hookimpl from pytask_stata.shared import convert_task_id_to_name_of_log_file -from pytask_stata.shared import get_node_from_dictionary from pytask_stata.shared import STATA_COMMANDS @hookimpl def pytask_execute_task_setup(session, task): """Check if Stata is found on the PATH.""" - if ( - get_specific_markers_from_task(task, "stata") - and session.config["stata"] is None - ): + if has_mark(task, "stata") and session.config["stata"] is None: raise RuntimeError( "Stata is needed to run do-files, but it is not found on your PATH.\n\n" f"We are looking for one of {STATA_COMMANDS} on your PATH. If you have a" @@ -36,15 +32,13 @@ def pytask_execute_task_teardown(session, task): or ``r(601)``. """ - if get_specific_markers_from_task(task, "stata"): + if has_mark(task, "stata"): if session.config["platform"] == "win32": - log_name = convert_task_id_to_name_of_log_file(task.name) + log_name = convert_task_id_to_name_of_log_file(task.short_name) path_to_log = task.path.with_name(log_name).with_suffix(".log") else: - source = get_node_from_dictionary( - task.depends_on, session.config["stata_source_key"] - ) - path_to_log = source.path.with_suffix(".log") + node = task.depends_on["__script"] + path_to_log = node.path.with_suffix(".log") n_lines = session.config["stata_check_log_lines"] diff --git a/src/pytask_stata/parametrize.py b/src/pytask_stata/parametrize.py index 6d73abd..300ee31 100644 --- a/src/pytask_stata/parametrize.py +++ b/src/pytask_stata/parametrize.py @@ -1,8 +1,8 @@ """Parametrize tasks.""" from __future__ import annotations -from _pytask.config import hookimpl -from _pytask.mark import MARK_GEN as mark # noqa: N811 +import pytask +from pytask import hookimpl @hookimpl @@ -10,4 +10,4 @@ def pytask_parametrize_kwarg_to_marker(obj, kwargs): """Attach parametrized stata arguments to the function with a marker.""" if callable(obj): if "stata" in kwargs: - mark.stata(kwargs.pop("stata"))(obj) + pytask.mark.stata(**kwargs.pop("stata"))(obj) diff --git a/src/pytask_stata/plugin.py b/src/pytask_stata/plugin.py index 686e412..b57f5db 100644 --- a/src/pytask_stata/plugin.py +++ b/src/pytask_stata/plugin.py @@ -1,7 +1,7 @@ """Register hook specifications and implementations.""" from __future__ import annotations -from _pytask.config import hookimpl +from pytask import hookimpl from pytask_stata import cli from pytask_stata import collect from pytask_stata import config diff --git a/src/pytask_stata/shared.py b/src/pytask_stata/shared.py index 49ef00e..a188e18 100644 --- a/src/pytask_stata/shared.py +++ b/src/pytask_stata/shared.py @@ -2,6 +2,9 @@ from __future__ import annotations import sys +from pathlib import Path +from typing import Iterable +from typing import Sequence if sys.platform == "darwin": @@ -34,6 +37,59 @@ STATA_COMMANDS = [] +_ERROR_MSG = """The old syntax for @pytask.mark.stata was suddenly deprecated starting \ +with pytask-stata v0.2 to provide a better user experience. Thank you for your \ +understanding! + +It is recommended to upgrade to the new syntax, so you enjoy all the benefits of v0.2 \ +of pytask and pytask-stata. + +You can find a manual here: \ +https://github.com/pytask-dev/pytask-stata/blob/v0.2.0/README.md + +Upgrading can be as easy as rewriting your current task from + + @pytask.mark.stata(["--option", "path_to_dependency.txt"]) + @pytask.mark.depends_on("script.do") + @pytask.mark.produces("out.csv") + def task_r(): + ... + +to + + @pytask.mark.stata(script="script.do", options="--option") + @pytask.mark.depends_on("path_to_dependency.txt") + @pytask.mark.produces("out.csv") + def task_r(): + ... + +You can also fix the version of pytask and pytask-stata to <0.2, so you do not have to \ +to upgrade. At the same time, you will not enjoy the improvements released with \ +version v0.2 of pytask and pytask-stata. + +""" + + +def stata( + *args, + script: str | Path | None = None, + options: str | Iterable[str] | None = None, +) -> tuple[str | Path | None, str | Iterable[str] | None]: + """Specify command line options for Stata. + + Parameters + ---------- + options : str | Iterable[str] | None + One or multiple command line options passed to Stata. + + """ + if args or script is None: + raise RuntimeError(_ERROR_MSG) + + options = [] if options is None else list(map(str, _to_list(options))) + return script, options + + def convert_task_id_to_name_of_log_file(id_): """Convert task to id to name of log file. @@ -43,15 +99,15 @@ def convert_task_id_to_name_of_log_file(id_): .. code-block:: none - C:/task_dummy.py::task_dummy[arg1] -> task_dummy.log + C:/task_example.py::task_example[arg1] -> task_example.log This function creates a new id starting from the task module and by replacing dots and double colons with underscores. Example ------- - >>> convert_task_id_to_name_of_log_file("C:/task_dummy.py::task_dummy[arg1]") - 'task_dummy_py_task_dummy[arg1]' + >>> convert_task_id_to_name_of_log_file("C:/task_example.py::task_example[arg1]") + 'task_example_py_task_example[arg1]' """ id_without_parent_directories = id_.rsplit("/")[-1] @@ -59,7 +115,27 @@ def convert_task_id_to_name_of_log_file(id_): return converted_id -def get_node_from_dictionary(obj, key, fallback=0): - if isinstance(obj, dict): - obj = obj.get(key) or obj.get(fallback) - return obj +def _to_list(scalar_or_iter): + """Convert scalars and iterables to list. + + Parameters + ---------- + scalar_or_iter : str or list + + Returns + ------- + list + + Examples + -------- + >>> _to_list("a") + ['a'] + >>> _to_list(["b"]) + ['b'] + + """ + return ( + [scalar_or_iter] + if isinstance(scalar_or_iter, str) or not isinstance(scalar_or_iter, Sequence) + else list(scalar_or_iter) + ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_collect.py b/tests/test_collect.py index e285031..304ae51 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -1,150 +1,76 @@ from __future__ import annotations from contextlib import ExitStack as does_not_raise # noqa: N813 -from pathlib import Path import pytest -from _pytask.mark import Mark -from _pytask.nodes import FilePathNode -from pytask_stata.collect import _merge_all_markers -from pytask_stata.collect import _prepare_cmd_options -from pytask_stata.collect import pytask_collect_task_teardown +from pytask import Mark +from pytask_stata.collect import _parse_stata_mark from pytask_stata.collect import stata -from pytask_stata.shared import get_node_from_dictionary - - -class DummyClass: - pass - - -def task_dummy(): - pass - - -@pytest.mark.unit -@pytest.mark.parametrize( - "stata_args, expected", - [ - (None, []), - ("some-arg", ["some-arg"]), - (["arg1", "arg2"], ["arg1", "arg2"]), - ], -) -def test_stata(stata_args, expected): - options = stata(stata_args) - assert options == expected @pytest.mark.unit @pytest.mark.parametrize( - "marks, expected", + "args, kwargs, expectation, expected", [ + ((), {}, pytest.raises(RuntimeError, match="The old syntax"), None), ( - [Mark("stata", ("a",), {}), Mark("stata", ("b",), {})], - Mark("stata", ("a", "b"), {}), + ("-o"), + {"script": "script.do"}, + pytest.raises(RuntimeError, match="The old syntax"), + None, ), ( - [Mark("stata", ("a",), {}), Mark("stata", (), {"stata": "b"})], - Mark("stata", ("a",), {"stata": "b"}), + (), + {"options": ("-o")}, + pytest.raises(RuntimeError, match="The old syntax"), + None, + ), + ( + (), + {"script": "script.do", "options": "--option"}, + does_not_raise(), + ("script.do", ["--option"]), + ), + ( + (), + {"script": "script.do", "options": [1]}, + does_not_raise(), + ("script.do", ["1"]), ), ], ) -def test_merge_all_markers(marks, expected): - task = DummyClass() - task.markers = marks - out = _merge_all_markers(task) - assert out == expected - - -@pytest.mark.unit -@pytest.mark.parametrize( - "args", - [ - [], - ["a"], - ["a", "b"], - ], -) -@pytest.mark.parametrize("stata_source_key", ["source", "do"]) -@pytest.mark.parametrize("platform", ["win32", "linux", "darwin"]) -def test_prepare_cmd_options(args, stata_source_key, platform): - session = DummyClass() - session.config = { - "stata": "stata", - "stata_source_key": stata_source_key, - "platform": platform, - } - - node = DummyClass() - node.path = Path("script.do") - task = DummyClass() - task.depends_on = {stata_source_key: node} - task.name = "task" - - result = _prepare_cmd_options(session, task, args) - - expected = [ - "stata", - "-e", - "do", - "script.do", - *args, - ] - if platform == "win32": - expected.append("-task") - - assert result == expected +def test_stata(args, kwargs, expectation, expected): + with expectation: + options = stata(*args, **kwargs) + assert options == expected @pytest.mark.unit @pytest.mark.parametrize( - "depends_on, produces, expectation", + "mark, expectation, expected", [ - (["script.do"], ["any_out.dta"], does_not_raise()), - (["script.txt"], ["any_out.dta"], pytest.raises(ValueError)), - (["input.dta", "script.do"], ["any_out.dta"], pytest.raises(ValueError)), + ( + Mark("stata", (), {}), + pytest.raises(RuntimeError, match="The old syntax for @pytask.mark.stata"), + Mark("stata", (), {"script": None, "options": []}), + ), + ( + Mark("stata", ("-o"), {}), + pytest.raises(RuntimeError, match="The old syntax for @pytask.mark.stata"), + None, + ), + ( + Mark("stata", (), {"script": "script.do"}), + does_not_raise(), + Mark("stata", (), {"script": "script.do", "options": []}), + ), ], ) -@pytest.mark.parametrize("platform", ["win32", "darwin", "linux"]) -def test_pytask_collect_task_teardown( - tmp_path, depends_on, produces, platform, expectation +def test_parse_stata_mark( + mark, + expectation, + expected, ): - session = DummyClass() - session.config = { - "stata": "stata", - "stata_source_key": "source", - "platform": platform, - } - - task = DummyClass() - task.depends_on = { - i: FilePathNode.from_path(tmp_path / n) for i, n in enumerate(depends_on) - } - task.produces = { - i: FilePathNode.from_path(tmp_path / n) for i, n in enumerate(produces) - } - task.function = task_dummy - task.name = "task_dummy" - task.path = Path() - - markers = [Mark("stata", (), {})] - task.markers = markers - task.function.pytaskmark = markers - with expectation: - pytask_collect_task_teardown(session, task) - - -@pytest.mark.unit -@pytest.mark.parametrize( - "obj, key, expected", - [ - (1, "asds", 1), - (1, None, 1), - ({"a": 1}, "a", 1), - ({0: 1}, "a", 1), - ], -) -def test_get_node_from_dictionary(obj, key, expected): - result = get_node_from_dictionary(obj, key) - assert result == expected + out = _parse_stata_mark(mark) + assert out == expected diff --git a/tests/test_execute.py b/tests/test_execute.py index 2d8025d..5fb4a1b 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -3,17 +3,19 @@ import sys import textwrap from contextlib import ExitStack as does_not_raise # noqa: N813 +from pathlib import Path import pytest -from _pytask.mark import Mark -from conftest import needs_stata +from pytask import cli +from pytask import ExitCode from pytask import main +from pytask import Mark +from pytask import Session +from pytask import Task from pytask_stata.config import STATA_COMMANDS from pytask_stata.execute import pytask_execute_task_setup - -class DummyClass: - pass +from tests.conftest import needs_stata @pytest.mark.unit @@ -26,11 +28,14 @@ class DummyClass: def test_pytask_execute_task_setup_raise_error(stata, platform, expectation): """Make sure that the task setup raises errors.""" # Act like r is installed since we do not test this. - task = DummyClass() - task.markers = [Mark("stata", (), {})] + task = Task( + base_name="task_example", + path=Path(), + function=None, + markers=[Mark("stata", (), {})], + ) - session = DummyClass() - session.config = {"stata": stata, "platform": platform} + session = Session(config={"stata": stata, "platform": platform}) with expectation: pytask_execute_task_setup(session, task) @@ -38,17 +43,16 @@ def test_pytask_execute_task_setup_raise_error(stata, platform, expectation): @needs_stata @pytest.mark.end_to_end -def test_run_do_file(tmp_path): +def test_run_do_file(runner, tmp_path): task_source = """ import pytask - @pytask.mark.stata - @pytask.mark.depends_on("script.do") + @pytask.mark.stata(script="script.do") @pytask.mark.produces("auto.dta") def task_run_do_file(): pass """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(task_source)) do_file = """ sysuse auto, clear @@ -56,13 +60,44 @@ def task_run_do_file(): """ tmp_path.joinpath("script.do").write_text(textwrap.dedent(do_file)) - session = main({"paths": tmp_path, "stata_keep_log": True}) + result = runner.invoke(cli, [tmp_path.as_posix(), "--stata-keep-log"]) - assert session.exit_code == 0 + assert result.exit_code == ExitCode.OK assert tmp_path.joinpath("auto.dta").exists() if sys.platform == "win32": - assert tmp_path.joinpath("task_dummy_py_task_run_do_file.log").exists() + assert tmp_path.joinpath("task_example_py_task_run_do_file.log").exists() + else: + assert tmp_path.joinpath("script.log").exists() + + +@needs_stata +@pytest.mark.end_to_end +def test_run_do_file_w_task_decorator(runner, tmp_path): + task_source = """ + import pytask + + @pytask.mark.task + @pytask.mark.stata(script="script.do") + @pytask.mark.produces("auto.dta") + def run_do_file(): + pass + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(task_source)) + + do_file = """ + sysuse auto, clear + save auto + """ + tmp_path.joinpath("script.do").write_text(textwrap.dedent(do_file)) + + result = runner.invoke(cli, [tmp_path.as_posix(), "--stata-keep-log"]) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("auto.dta").exists() + + if sys.platform == "win32": + assert tmp_path.joinpath("task_example_py_run_do_file.log").exists() else: assert tmp_path.joinpath("script.log").exists() @@ -72,14 +107,12 @@ def test_raise_error_if_stata_is_not_found(tmp_path, monkeypatch): task_source = """ import pytask - @pytask.mark.stata - @pytask.mark.depends_on("script.do") + @pytask.mark.stata(script="script.do") @pytask.mark.produces("out.dta") def task_run_do_file(): pass - """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(task_source)) tmp_path.joinpath("script.do").write_text(textwrap.dedent("1 + 1")) # Hide Stata if available. @@ -89,25 +122,23 @@ def task_run_do_file(): session = main({"paths": tmp_path}) - assert session.exit_code == 1 + assert session.exit_code == ExitCode.FAILED assert isinstance(session.execution_reports[0].exc_info[1], RuntimeError) @needs_stata @pytest.mark.end_to_end -def test_run_do_file_w_wrong_cmd_option(tmp_path): +def test_run_do_file_w_wrong_cmd_option(runner, tmp_path): """Apparently, Stata simply discards wrong cmd options.""" task_source = """ import pytask - @pytask.mark.stata("--wrong-flag") - @pytask.mark.depends_on("script.do") + @pytask.mark.stata(script="script.do", options="--wrong-flag") @pytask.mark.produces("out.dta") def task_run_do_file(): pass - """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(task_source)) do_file = """ sysuse auto, clear @@ -115,26 +146,25 @@ def task_run_do_file(): """ tmp_path.joinpath("script.do").write_text(textwrap.dedent(do_file)) - session = main({"paths": tmp_path}) + result = runner.invoke(cli, [tmp_path.as_posix()]) - assert session.exit_code == 0 + assert result.exit_code == ExitCode.OK @needs_stata @pytest.mark.end_to_end -def test_run_do_file_by_passing_path(tmp_path): +def test_run_do_file_by_passing_path(runner, tmp_path): """Replicates example under "Command Line Arguments" in Readme.""" task_source = """ import pytask from pathlib import Path - @pytask.mark.stata(Path(__file__).parent / "auto.dta") - @pytask.mark.depends_on("script.do") + @pytask.mark.stata(script="script.do", options=Path(__file__).parent / "auto.dta") @pytask.mark.produces("auto.dta") def task_run_do_file(): pass """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(task_source)) do_file = """ args produces @@ -143,6 +173,26 @@ def task_run_do_file(): """ tmp_path.joinpath("script.do").write_text(textwrap.dedent(do_file)) - session = main({"paths": tmp_path}) + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + + +@needs_stata +@pytest.mark.end_to_end +def test_run_do_file_fails_with_multiple_marks(runner, tmp_path): + task_source = """ + import pytask + + @pytask.mark.stata(script="script.do") + @pytask.mark.stata(script="script.do") + @pytask.mark.produces("auto.dta") + def task_run_do_file(): + pass + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(task_source)) + tmp_path.joinpath("script.do").touch() + + result = runner.invoke(cli, [tmp_path.as_posix(), "--stata-keep-log"]) - assert session.exit_code == 0 + assert result.exit_code == ExitCode.COLLECTION_FAILED + assert "has multiple @pytask.mark.stata marks" in result.output diff --git a/tests/test_normal_execution_w_plugin.py b/tests/test_normal_execution_w_plugin.py index ee19252..4bd592e 100644 --- a/tests/test_normal_execution_w_plugin.py +++ b/tests/test_normal_execution_w_plugin.py @@ -22,7 +22,7 @@ def test_execution_w_varying_dependencies_products( @pytask.mark.depends_on({dependencies}) @pytask.mark.produces({products}) - def task_dummy(depends_on, produces): + def task_example(depends_on, produces): if isinstance(produces, dict): produces = produces.values() elif isinstance(produces, Path): @@ -30,7 +30,7 @@ def task_dummy(depends_on, produces): for product in produces: product.touch() """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) for dependency in dependencies: tmp_path.joinpath(dependency).touch() diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 2ca16f4..b7044cf 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -5,8 +5,10 @@ import time import pytest -from conftest import needs_stata from pytask import cli +from pytask import ExitCode + +from tests.conftest import needs_stata try: import pytask_parallel # noqa: F401 @@ -23,18 +25,59 @@ @needs_stata @pytest.mark.end_to_end -def test_parallel_parametrization_over_source_files(runner, tmp_path): +def test_parallel_parametrization_over_source_files_w_parametrize(runner, tmp_path): source = """ import pytask - @pytask.mark.stata @pytask.mark.parametrize( - "depends_on, produces", [("script_1.do", "1.dta"), ("script_2.do", "2.dta")] + "stata, produces", [( + {"script": "script_1.do"}, "1.dta"), ({"script": "script_2.do"}, "2.dta") + ] ) def task_execute_do_file(): pass """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + for i in range(1, 3): + do_file = f""" + sleep 4000 + sysuse auto, clear + save {i} + """ + tmp_path.joinpath(f"script_{i}.do").write_text(textwrap.dedent(do_file)) + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + duration_normal = time.time() - start + + for name in ["1.dta", "2.dta"]: + tmp_path.joinpath(name).unlink() + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix(), "-n", 2]) + assert result.exit_code == ExitCode.OK + duration_parallel = time.time() - start + + assert duration_parallel < duration_normal + + +@needs_stata +@pytest.mark.end_to_end +def test_parallel_parametrization_over_source_files_w_loop(runner, tmp_path): + source = """ + import pytask + + for i in range (1, 3): + + @pytask.mark.task + @pytask.mark.stata(script=f"script_{i}.do") + @pytask.mark.produces(f"{i}.dta") + def task_execute_do_file(): + pass + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) do_file = """ sleep 4000 @@ -52,7 +95,7 @@ def task_execute_do_file(): start = time.time() result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == 0 + assert result.exit_code == ExitCode.OK duration_normal = time.time() - start for name in ["1.dta", "2.dta"]: @@ -60,7 +103,7 @@ def task_execute_do_file(): start = time.time() result = runner.invoke(cli, [tmp_path.as_posix(), "-n", 2]) - assert result.exit_code == 0 + assert result.exit_code == ExitCode.OK duration_parallel = time.time() - start assert duration_parallel < duration_normal @@ -68,19 +111,61 @@ def task_execute_do_file(): @needs_stata @pytest.mark.end_to_end -def test_parallel_parametrization_over_source_file(runner, tmp_path): +def test_parallel_parametrization_over_source_file_w_parametrize(runner, tmp_path): source = """ import pytask - @pytask.mark.depends_on("script.do") @pytask.mark.parametrize( "produces, stata", - [("output_1.dta", ("output_1",)), ("output_2.dta", ("output_2",))], + [ + ("output_1.dta", {"script": "script.do", "options": ("output_1",)}), + ("output_2.dta", {"script": "script.do", "options": ("output_2",)}) + ], ) def task_execute_do_file(): pass """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + do_file = """ + sleep 4000 + sysuse auto, clear + args produces + save "`produces'" + """ + tmp_path.joinpath("script.do").write_text(textwrap.dedent(do_file)) + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + duration_normal = time.time() - start + + for name in ["output_1.dta", "output_2.dta"]: + tmp_path.joinpath(name).unlink() + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix(), "-n", 2]) + assert result.exit_code == ExitCode.OK + duration_parallel = time.time() - start + + assert duration_parallel < duration_normal + + +@needs_stata +@pytest.mark.end_to_end +def test_parallel_parametrization_over_source_file_w_loop(runner, tmp_path): + source = """ + import pytask + + for i in range (1, 3): + + @pytask.mark.task + @pytask.mark.stata(script="script.do", options=f"output_{i}") + @pytask.mark.produces(f"output_{i}.dta") + def task_execute_do_file(): + pass + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) do_file = """ sleep 4000 @@ -92,7 +177,7 @@ def task_execute_do_file(): start = time.time() result = runner.invoke(cli, [tmp_path.as_posix()]) - assert result.exit_code == 0 + assert result.exit_code == ExitCode.OK duration_normal = time.time() - start for name in ["output_1.dta", "output_2.dta"]: @@ -100,7 +185,7 @@ def task_execute_do_file(): start = time.time() result = runner.invoke(cli, [tmp_path.as_posix(), "-n", 2]) - assert result.exit_code == 0 + assert result.exit_code == ExitCode.OK duration_parallel = time.time() - start assert duration_parallel < duration_normal diff --git a/tests/test_parametrize.py b/tests/test_parametrize.py index ca34019..917f460 100644 --- a/tests/test_parametrize.py +++ b/tests/test_parametrize.py @@ -4,60 +4,89 @@ import textwrap import pytest -from conftest import needs_stata -from pytask import main +from pytask import cli +from pytask import ExitCode + +from tests.conftest import needs_stata @needs_stata @pytest.mark.end_to_end -def test_parametrized_execution_of_do_file(tmp_path): +def test_parametrized_execution_of_do_file_w_parametrize(runner, tmp_path): task_source = """ import pytask - @pytask.mark.stata - @pytask.mark.parametrize("depends_on, produces", [ - ("script_1.do", "0.dta"), - ("script_2.do", "1.dta"), - ]) - def task_run_do_file(): + @pytask.mark.parametrize( + "stata, produces", [( + {"script": "script_1.do"}, "1.dta"), ({"script": "script_2.do"}, "2.dta") + ] + ) + def task_execute_do_file(): pass """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(task_source)) - for name, out in [ - ("script_1.do", "0"), - ("script_2.do", "1"), - ]: + for i in range(1, 3): do_file = f""" sysuse auto, clear - save {out} + save {i} """ - tmp_path.joinpath(name).write_text(textwrap.dedent(do_file)) + tmp_path.joinpath(f"script_{i}.do").write_text(textwrap.dedent(do_file)) - session = main({"paths": tmp_path}) + result = runner.invoke(cli, [tmp_path.as_posix()]) - assert session.exit_code == 0 - assert tmp_path.joinpath("0.dta").exists() + assert result.exit_code == ExitCode.OK assert tmp_path.joinpath("1.dta").exists() + assert tmp_path.joinpath("2.dta").exists() @needs_stata @pytest.mark.end_to_end -def test_parametrize_command_line_options(tmp_path): - task_source = """ +def test_parametrized_execution_of_do_file_w_loop(runner, tmp_path): + source = """ import pytask - from pathlib import Path - SRC = Path(__file__).parent + for i in range (1, 3): - @pytask.mark.depends_on("script.do") - @pytask.mark.parametrize("produces, stata", [ - (SRC / "0.dta", SRC / "0.dta"), (SRC / "1.dta", SRC / "1.dta"), - ]) + @pytask.mark.task + @pytask.mark.stata(script=f"script_{i}.do") + @pytask.mark.produces(f"{i}.dta") + def task_execute_do_file(): + pass + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + for i in range(1, 3): + do_file = f""" + sysuse auto, clear + save {i} + """ + tmp_path.joinpath(f"script_{i}.do").write_text(textwrap.dedent(do_file)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("1.dta").exists() + assert tmp_path.joinpath("2.dta").exists() + + +@needs_stata +@pytest.mark.end_to_end +def test_parametrize_command_line_options_w_parametrize(runner, tmp_path): + task_source = """ + import pytask + + @pytask.mark.parametrize( + "produces, stata", + [ + ("output_1.dta", {"script": "script.do", "options": ("output_1",)}), + ("output_2.dta", {"script": "script.do", "options": ("output_2",)}) + ], + ) def task_execute_do_file(): pass """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(task_source)) + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(task_source)) latex_source = """ sysuse auto, clear @@ -66,19 +95,56 @@ def task_execute_do_file(): """ tmp_path.joinpath("script.do").write_text(textwrap.dedent(latex_source)) - session = main({"paths": tmp_path, "stata_keep_log": True}) + result = runner.invoke(cli, [tmp_path.as_posix(), "--stata-keep-log"]) - assert session.exit_code == 0 - assert tmp_path.joinpath("0.dta").exists() - assert tmp_path.joinpath("1.dta").exists() + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("output_1.dta").exists() + assert tmp_path.joinpath("output_2.dta").exists() # Test that log files with different names are produced. if sys.platform == "win32": assert tmp_path.joinpath( - "task_dummy_py_task_execute_do_file[produces0-stata0].log" + "task_example_py_task_execute_do_file[output_1_dta-stata0].log" ).exists() assert tmp_path.joinpath( - "task_dummy_py_task_execute_do_file[produces1-stata1].log" + "task_example_py_task_execute_do_file[output_2_dta-stata1].log" ).exists() else: assert tmp_path.joinpath("script.log").exists() + + +@needs_stata +@pytest.mark.end_to_end +def test_parametrize_command_line_options_w_loop(runner, tmp_path): + task_source = """ + import pytask + + for i in range (1, 3): + + @pytask.mark.task + @pytask.mark.stata(script="script.do", options=f"output_{i}") + @pytask.mark.produces(f"output_{i}.dta") + def task_execute_do_file(): + pass + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(task_source)) + + latex_source = """ + sysuse auto, clear + args produces + save "`produces'" + """ + tmp_path.joinpath("script.do").write_text(textwrap.dedent(latex_source)) + + result = runner.invoke(cli, [tmp_path.as_posix(), "--stata-keep-log"]) + + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("output_1.dta").exists() + assert tmp_path.joinpath("output_2.dta").exists() + + # Test that log files with different names are produced. + if sys.platform == "win32": + assert tmp_path.joinpath("task_example_py_task_execute_do_file[0].log").exists() + assert tmp_path.joinpath("task_example_py_task_execute_do_file[1].log").exists() + else: + assert tmp_path.joinpath("script.log").exists() diff --git a/tox.ini b/tox.ini index b3e118e..da168af 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = pytest, pre-commit +envlist = pytest skipsdist = True skip_missing_interpreters = True passenv = PY_IGNORE_IMPORTMISMATCH @@ -13,7 +13,7 @@ conda_channels = conda-forge nodefaults conda_deps = - pytask >=0.1.0 + pytask >=0.2 pytest pytest-cov pytest-xdist @@ -21,14 +21,6 @@ commands = pip install --no-deps -e . pytest {posargs} -[testenv:pre-commit] -deps = pre-commit -commands = pre-commit run --all-files - -[doc8] -ignore = D002, D004 -max-line-length = 89 - [flake8] docstring-convention = numpy ignore =