From 25304ee251aff8363d09fbfab42a532a5a11fcab Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Mon, 9 Jan 2023 23:42:09 +0100 Subject: [PATCH 1/2] Deprecate INI configurations and align with pytask v0.3. --- .github/workflows/main.yml | 2 +- CHANGES.md | 4 +++ Manifest.toml | 41 +++++++++++++++++++++------- README.md | 15 ++++++++++- environment.yml | 4 +-- setup.cfg | 2 +- src/pytask_julia/collect.py | 6 ++--- src/pytask_julia/config.py | 28 ++++++++++++-------- src/pytask_julia/serialization.py | 10 +++---- src/pytask_julia/shared.py | 44 ++----------------------------- tests/test_collect.py | 32 ++-------------------- tests/test_shared.py | 13 --------- tox.ini | 4 +-- 13 files changed, 84 insertions(+), 121 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7a0d831..42228a9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,7 @@ jobs: - name: Install core dependencies. shell: bash -l {0} - run: conda install -c conda-forge tox-conda coverage mamba + run: mamba install -c conda-forge tox-conda coverage mamba # Unit, integration, and end-to-end tests. diff --git a/CHANGES.md b/CHANGES.md index c361034..b3b1015 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,10 @@ chronological order. Releases follow [semantic versioning](https://semver.org/) releases are available on [PyPI](https://pypi.org/project/pytask-julia) and [Anaconda.org](https://anaconda.org/conda-forge/pytask-julia). +## 0.3.0 - 2023-xx-xx + +- {pull}`17` deprecates INI configurations and aligns the package with pytask v0.3. + ## 0.2.1 - 2022-04-16 - {pull}`12` includes `pyproject.toml` in the package. diff --git a/Manifest.toml b/Manifest.toml index 702d6d5..140e3dd 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,10 +1,12 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.7.1" +julia_version = "1.8.5" manifest_format = "2.0" +project_hash = "df1f4ff6ff5c9fd1541ff7a0d1e10d339e63d6ac" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" @@ -19,6 +21,7 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" @@ -42,10 +45,12 @@ version = "0.21.3" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" [[deps.LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] @@ -54,15 +59,16 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [[deps.Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778" +git-tree-sha1 = "c7cb1f5d892775ba13767a87c7ada0b980ea0a71" uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" -version = "1.16.1+1" +version = "1.16.1+2" [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" @@ -74,31 +80,35 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.0+0" [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.2.1" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" [[deps.Parsers]] -deps = ["Dates"] -git-tree-sha1 = "13468f237353112a01b2d6b32f3d0f80219944aa" +deps = ["Dates", "SnoopPrecompile"] +git-tree-sha1 = "6466e524967496866901a78fca3f2e9ea445a559" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.2.2" +version = "2.5.2" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.8.0" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "2cf929d64681236a2e074ffafb8d568733d2e6af" +git-tree-sha1 = "47e5f437cc0e7ef2ce8406ce1e7e24d44915f88d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.2.3" +version = "1.3.0" [[deps.Printf]] deps = ["Unicode"] @@ -114,10 +124,16 @@ uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +[[deps.SnoopPrecompile]] +git-tree-sha1 = "f604441450a3c0569830946e5b33b78c928e1a85" +uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" +version = "1.0.1" + [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" @@ -130,10 +146,12 @@ version = "0.3.5" [[deps.TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.0" [[deps.Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.1" [[deps.UUIDs]] deps = ["Random", "SHA"] @@ -144,18 +162,21 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[deps.YAML]] deps = ["Base64", "Dates", "Printf", "StringEncodings"] -git-tree-sha1 = "3c6e8b9f5cdaaa21340f841653942e1a6b6561e5" +git-tree-sha1 = "dbc7f1c0012a69486af79c8bcdb31be820670ba2" uuid = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" -version = "0.4.7" +version = "0.4.8" [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.12+3" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/README.md b/README.md index 6db1645..d85f95d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![image](https://img.shields.io/conda/vn/conda-forge/pytask-julia.svg)](https://anaconda.org/conda-forge/pytask-julia) [![image](https://img.shields.io/conda/pn/conda-forge/pytask-julia.svg)](https://anaconda.org/conda-forge/pytask-julia) [![PyPI - License](https://img.shields.io/pypi/l/pytask-julia)](https://pypi.org/project/pytask-julia) -[![image](https://img.shields.io/github/workflow/status/pytask-dev/pytask-julia/main/main)](https://github.com/pytask-dev/pytask-julia/actions?query=branch%3Amain) +[![image](https://img.shields.io/github/actions/workflow/status/pytask-dev/pytask-julia/main.yml?branch=main)](https://github.com/pytask-dev/pytask-julia/actions?query=branch%3Amain) [![image](https://codecov.io/gh/pytask-dev/pytask-julia/branch/main/graph/badge.svg)](https://codecov.io/gh/pytask-dev/pytask-julia) [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pytask-dev/pytask-julia/main.svg)](https://results.pre-commit.ci/latest/github/pytask-dev/pytask-julia/main) [![image](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) @@ -296,3 +296,16 @@ julia_project = "environment" ## Changes Consult the [release notes](CHANGES.md) to find out about what is new. + +## Development + +After setting up the environment, enter + +```console +$ julia +$ ] +$ pkg> activate . +$ pkg> instantiate +``` + +to set up the Julia environment. diff --git a/environment.yml b/environment.yml index 597a467..0dc6323 100644 --- a/environment.yml +++ b/environment.yml @@ -12,8 +12,8 @@ dependencies: # Package dependencies - julia - - pytask >=0.2 - - pytask-parallel >=0.1 + - pytask >=0.3 + - pytask-parallel >=0.3 # Optional package dependencies - pyyaml diff --git a/setup.cfg b/setup.cfg index 5a5eb73..70bac67 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,7 +25,7 @@ project_urls = packages = find: install_requires = pybaum>=0.1.1 - pytask>=0.2 + pytask>=0.3 python_requires = >=3.7 include_package_data = True package_dir = =src diff --git a/src/pytask_julia/collect.py b/src/pytask_julia/collect.py index 4149cdb..aaee6bd 100644 --- a/src/pytask_julia/collect.py +++ b/src/pytask_julia/collect.py @@ -14,7 +14,7 @@ from pytask import produces from pytask import remove_marks from pytask import Task -from pytask_julia.serialization import SERIALIZER +from pytask_julia.serialization import SERIALIZERS from pytask_julia.shared import julia from pytask_julia.shared import parse_relative_path @@ -122,9 +122,9 @@ def _parse_julia_mark( if ( isinstance(parsed_kwargs["serializer"], str) - and parsed_kwargs["serializer"] in SERIALIZER + and parsed_kwargs["serializer"] in SERIALIZERS ): - proposed_suffix = SERIALIZER[parsed_kwargs["serializer"]]["suffix"] + proposed_suffix = SERIALIZERS[parsed_kwargs["serializer"]]["suffix"] else: proposed_suffix = default_suffix parsed_kwargs["suffix"] = suffix if suffix else proposed_suffix diff --git a/src/pytask_julia/config.py b/src/pytask_julia/config.py index 9c1ca80..5b2e57d 100644 --- a/src/pytask_julia/config.py +++ b/src/pytask_julia/config.py @@ -1,33 +1,39 @@ """Configure pytask.""" from __future__ import annotations +from typing import Any + from pytask import hookimpl +from pytask_julia.serialization import SERIALIZERS from pytask_julia.shared import parse_relative_path @hookimpl -def pytask_parse_config(config, config_from_file): +def pytask_parse_config(config): """Register the julia marker.""" config["markers"]["julia"] = "Tasks which are executed with Julia." - config["julia_serializer"] = config_from_file.get("julia_serializer", "json") - config["julia_suffix"] = config_from_file.get("julia_suffix", "") + config["julia_serializer"] = config.get("julia_serializer", "json") + if config["julia_serializer"] not in SERIALIZERS: + raise ValueError( + f"'julia_serializer' is {config['julia_serializer']} and not one of " + f"{list(SERIALIZERS)}." + ) + config["julia_suffix"] = config.get("julia_suffix", "") config["julia_options"] = _parse_value_or_whitespace_option( - config_from_file.get("julia_options") + config.get("julia_options") ) - project = config_from_file.get("julia_project") + project = config.get("julia_project") if project is None: config["julia_project"] = project else: config["julia_project"] = parse_relative_path(project, config["root"]) -def _parse_value_or_whitespace_option(value: str | None) -> None | str | list[str]: +def _parse_value_or_whitespace_option(value: Any) -> None | list[str]: """Parse option which can hold a single value or values separated by new lines.""" - if value in ["none", "None", None, ""]: + if value is None: return None - elif isinstance(value, list): + if isinstance(value, list): return list(map(str, value)) - elif isinstance(value, str): - return [v.strip() for v in value.split(" ") if v.strip()] else: - raise ValueError(f"Input {value!r} is neither a 'str' nor 'None'.") + raise ValueError(f"'julia_options' is {value} and not a list.") diff --git a/src/pytask_julia/serialization.py b/src/pytask_julia/serialization.py index f11d30e..57c923d 100644 --- a/src/pytask_julia/serialization.py +++ b/src/pytask_julia/serialization.py @@ -12,7 +12,7 @@ _HIDDEN_FOLDER = ".pytask" -SERIALIZER = { +SERIALIZERS = { "json": {"serializer": json.dumps, "suffix": ".json"}, } @@ -21,8 +21,8 @@ except ImportError: # pragma: no cover pass else: - SERIALIZER["yaml"] = {"serializer": yaml.dump, "suffix": ".yaml"} - SERIALIZER["yml"] = {"serializer": yaml.dump, "suffix": ".yml"} + SERIALIZERS["yaml"] = {"serializer": yaml.dump, "suffix": ".yaml"} + SERIALIZERS["yml"] = {"serializer": yaml.dump, "suffix": ".yml"} def create_path_to_serialized(task: Task, suffix: str) -> Path: @@ -57,8 +57,8 @@ def serialize_keyword_arguments( ) -> None: if callable(serializer): serializer_func = serializer - elif isinstance(serializer, str) and serializer in SERIALIZER: - serializer_func = SERIALIZER[serializer]["serializer"] + elif isinstance(serializer, str) and serializer in SERIALIZERS: + serializer_func = SERIALIZERS[serializer]["serializer"] else: # pragma: no cover raise ValueError(f"Serializer {serializer!r} is not known.") diff --git a/src/pytask_julia/shared.py b/src/pytask_julia/shared.py index 75e0257..95a15b9 100644 --- a/src/pytask_julia/shared.py +++ b/src/pytask_julia/shared.py @@ -6,45 +6,8 @@ from typing import Sequence -_ERROR_MSG = """The old syntax for @pytask.mark.julia was suddenly deprecated starting \ -with pytask-julia v0.2 to provide a better user experience. Thank you for your \ -understanding! - -It is recommended to upgrade to the new syntax, so you enjoy all the benefits of v0.2 of -pytask and pytask-julia which are - -- Access to 'depends_on' and 'produces', and other kwargs inside the Julia script. -- Support for Julia environments for running scripts with certain packages. - -You can find a manual here: \ -https://github.com/pytask-dev/pytask-julia/blob/v0.2.0/README.md - -Upgrading can be as easy as rewriting your current task from - - @pytask.mark.julia(["--threads", 2, "--", "path_to_dependency.txt"]) - @pytask.mark.depends_on("script.jl") - @pytask.mark.produces("out.csv") - def task_julia(): - ... - -to - - @pytask.mark.julia(script="script.jl", options=("--threads", 2)) - @pytask.mark.depends_on("path_to_dependency.txt") - @pytask.mark.produces("out.csv") - def task_julia(): - ... - -You can also fix the version of pytask and pytask-julia to <0.2, so you do not have to \ -to upgrade. At the same time, you will not enjoy the improvements released with \ -version v0.2 of pytask and pytask-julia. - -""" - - def julia( - *args, - script: str | Path | None = None, + script: str | Path, options: str | Iterable[str] | None = None, serializer: str | Callable[..., str] | str | None = None, suffix: str | None = None, @@ -60,7 +23,7 @@ def julia( Parameters ---------- - script : str | Path | None + script : str | Path The path to the Julia script which is executed. options : str | Iterable[str] | None One or multiple command line options passed to the interpreter for Julia. @@ -76,9 +39,6 @@ def julia( A path to an Julia environment used to execute this task. """ - if args or script is None: - raise RuntimeError(_ERROR_MSG) - options = [] if options is None else list(map(str, _to_list(options))) return script, options, serializer, suffix, project diff --git a/tests/test_collect.py b/tests/test_collect.py index dba48fd..953c1bc 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -6,7 +6,7 @@ from pytask import Mark from pytask_julia.collect import _parse_julia_mark from pytask_julia.collect import _parse_project -from pytask_julia.collect import SERIALIZER +from pytask_julia.collect import SERIALIZERS from tests.conftest import ROOT @@ -16,34 +16,6 @@ "mark, default_options, default_serializer, default_suffix, default_project, " "expectation, expected", [ - ( - Mark("julia", (), {}), - [], - None, - ".json", - None, - pytest.raises(RuntimeError, match="The old syntax for @pytask.mark.julia"), - Mark( - "julia", - (), - { - "script": None, - "options": [], - "serializer": None, - "suffix": ".json", - "project": None, - }, - ), - ), - ( - Mark("julia", ("-o"), {}), - [], - None, - ".json", - None, - pytest.raises(RuntimeError, match="The old syntax for @pytask.mark.julia"), - None, - ), ( Mark("julia", (), {"script": "script.jl"}), [], @@ -85,7 +57,7 @@ "script": "script.jl", "options": [], "serializer": "json", - "suffix": SERIALIZER["json"]["suffix"], + "suffix": SERIALIZERS["json"]["suffix"], "project": "some_path", }, ), diff --git a/tests/test_shared.py b/tests/test_shared.py index 96d959f..2525e08 100644 --- a/tests/test_shared.py +++ b/tests/test_shared.py @@ -10,19 +10,6 @@ @pytest.mark.parametrize( "args, kwargs, expectation, expected", [ - ((), {}, pytest.raises(RuntimeError, match="The old syntax"), None), - ( - ("-o"), - {"script": "script.jl"}, - pytest.raises(RuntimeError, match="The old syntax"), - None, - ), - ( - (), - {"options": ("-o")}, - pytest.raises(RuntimeError, match="The old syntax"), - None, - ), ( (), { diff --git a/tox.ini b/tox.ini index 0e7775b..e6bd452 100644 --- a/tox.ini +++ b/tox.ini @@ -16,8 +16,8 @@ conda_channels = nodefaults conda_deps = # Package dependencies - pytask >=0.2 - pytask-parallel >=0.1 + pytask >=0.3 + pytask-parallel >=0.3 # Optional package dependencies pyyaml From a298d134ad45f5665ae9216f2d5691bd6fe4f655 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Mon, 9 Jan 2023 23:44:07 +0100 Subject: [PATCH 2/2] to changes. --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b3b1015..a164ed6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,7 +7,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask-julia) and ## 0.3.0 - 2023-xx-xx -- {pull}`17` deprecates INI configurations and aligns the package with pytask v0.3. +- {pull}`18` deprecates INI configurations and aligns the package with pytask v0.3. ## 0.2.1 - 2022-04-16