From 7f048871bc1c332dd89ffceb84b4146d0c9e42b9 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 18 Apr 2024 23:26:35 +0200 Subject: [PATCH 1/3] Replace requests with httpx. --- .github/workflows/update-plugin-list.yml | 2 +- docs/source/developers_guide.md | 2 +- docs/source/how_to_guides/remote_files.md | 4 ++-- pyproject.toml | 3 --- requirements-dev.lock | 15 --------------- requirements.lock | 15 --------------- scripts/update_plugin_list.py | 6 +++--- tests/test_execute.py | 20 ++++++++++++++++++++ 8 files changed, 27 insertions(+), 40 deletions(-) diff --git a/.github/workflows/update-plugin-list.yml b/.github/workflows/update-plugin-list.yml index 252c6ddb8..bdef0e346 100644 --- a/.github/workflows/update-plugin-list.yml +++ b/.github/workflows/update-plugin-list.yml @@ -31,7 +31,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install packaging requests tabulate[widechars] tqdm + pip install packaging httpx tabulate[widechars] tqdm - name: Update Plugin List run: python scripts/update_plugin_list.py diff --git a/docs/source/developers_guide.md b/docs/source/developers_guide.md index 1a987b0c5..fbd06b77e 100644 --- a/docs/source/developers_guide.md +++ b/docs/source/developers_guide.md @@ -9,7 +9,7 @@ number of open file descriptors on Unix systems, which causes some tests and the the test suite to fail. If that happens, increase the limit with the following command. ```console -$ ulimit -n 4096 +ulimit -n 4096 ``` ## How to release diff --git a/docs/source/how_to_guides/remote_files.md b/docs/source/how_to_guides/remote_files.md index 6125af9c3..f43bd84c9 100644 --- a/docs/source/how_to_guides/remote_files.md +++ b/docs/source/how_to_guides/remote_files.md @@ -73,9 +73,9 @@ file as the ETag. If the file changes, so does the ETag, and pytask will detect Many files on the web do not provide an ETag like this version of the iris dataset. ```pycon ->>> import requests +>>> import httpx >>> url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data" ->>> r = requests.head(url) +>>> r = httpx.head(url) >>> r.headers {'Server': 'nginx/1.25.3', 'Date': 'Sun, 10 Dec 2023 23:59:21 GMT', 'Connection': 'keep-alive'} ``` diff --git a/pyproject.toml b/pyproject.toml index 5fcd58d4b..6c81c49c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,9 +68,6 @@ test = [ "pytest-cov", "pytest-xdist", "syrupy", - # For HTTPPath tests. - "aiohttp", - "requests", ] [project.urls] diff --git a/requirements-dev.lock b/requirements-dev.lock index 6b0a2800c..df7f9104f 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -8,10 +8,6 @@ # with-sources: false -e file:. -aiohttp==3.9.4 - # via pytask -aiosignal==1.3.1 - # via aiohttp alabaster==0.7.16 # via sphinx apeye==1.4.1 @@ -21,7 +17,6 @@ apeye-core==1.1.5 asttokens==2.4.1 # via stack-data attrs==23.2.0 - # via aiohttp # via jsonschema # via pytask # via referencing @@ -91,9 +86,6 @@ filelock==3.13.4 # via sphinx-toolbox fonttools==4.51.0 # via matplotlib -frozenlist==1.4.1 - # via aiohttp - # via aiosignal fsspec==2024.3.1 # via universal-pathlib furo==2024.1.29 @@ -105,7 +97,6 @@ html5lib==1.1 idna==3.7 # via apeye-core # via requests - # via yarl imagesize==1.4.1 # via sphinx iniconfig==2.0.0 @@ -160,9 +151,6 @@ mistune==3.0.2 # via nbconvert msgpack==1.0.8 # via cachecontrol -multidict==6.0.5 - # via aiohttp - # via yarl myst-parser==2.0.0 # via pytask natsort==8.4.0 @@ -258,7 +246,6 @@ referencing==0.34.0 requests==2.31.0 # via apeye # via cachecontrol - # via pytask # via sphinx rich==13.7.1 # via pytask @@ -364,5 +351,3 @@ webencodings==0.5.1 # via bleach # via html5lib # via tinycss2 -yarl==1.9.4 - # via aiohttp diff --git a/requirements.lock b/requirements.lock index 6b0a2800c..df7f9104f 100644 --- a/requirements.lock +++ b/requirements.lock @@ -8,10 +8,6 @@ # with-sources: false -e file:. -aiohttp==3.9.4 - # via pytask -aiosignal==1.3.1 - # via aiohttp alabaster==0.7.16 # via sphinx apeye==1.4.1 @@ -21,7 +17,6 @@ apeye-core==1.1.5 asttokens==2.4.1 # via stack-data attrs==23.2.0 - # via aiohttp # via jsonschema # via pytask # via referencing @@ -91,9 +86,6 @@ filelock==3.13.4 # via sphinx-toolbox fonttools==4.51.0 # via matplotlib -frozenlist==1.4.1 - # via aiohttp - # via aiosignal fsspec==2024.3.1 # via universal-pathlib furo==2024.1.29 @@ -105,7 +97,6 @@ html5lib==1.1 idna==3.7 # via apeye-core # via requests - # via yarl imagesize==1.4.1 # via sphinx iniconfig==2.0.0 @@ -160,9 +151,6 @@ mistune==3.0.2 # via nbconvert msgpack==1.0.8 # via cachecontrol -multidict==6.0.5 - # via aiohttp - # via yarl myst-parser==2.0.0 # via pytask natsort==8.4.0 @@ -258,7 +246,6 @@ referencing==0.34.0 requests==2.31.0 # via apeye # via cachecontrol - # via pytask # via sphinx rich==13.7.1 # via pytask @@ -364,5 +351,3 @@ webencodings==0.5.1 # via bleach # via html5lib # via tinycss2 -yarl==1.9.4 - # via aiohttp diff --git a/scripts/update_plugin_list.py b/scripts/update_plugin_list.py index cab04e183..56050e258 100644 --- a/scripts/update_plugin_list.py +++ b/scripts/update_plugin_list.py @@ -37,8 +37,8 @@ from textwrap import indent from typing import Generator +import httpx import packaging.version -import requests import tabulate import wcwidth from tqdm import tqdm @@ -87,7 +87,7 @@ def _escape_rst(text: str) -> str: def _iter_plugins() -> Generator[dict[str, str], None, None]: # noqa: C901 """Iterate over all plugins and format entries.""" regex = r">([\d\w-]*)" - response = requests.get("https://pypi.org/simple", timeout=20) + response = httpx.get("https://pypi.org/simple", timeout=20) matches = [ match @@ -98,7 +98,7 @@ def _iter_plugins() -> Generator[dict[str, str], None, None]: # noqa: C901 for match in tqdm(matches, smoothing=0): name = match.groups()[0] - response = requests.get(f"https://pypi.org/pypi/{name}/json", timeout=20) + response = httpx.get(f"https://pypi.org/pypi/{name}/json", timeout=20) if response.status_code == 404: # noqa: PLR2004 # Some packages might return a 404. continue diff --git a/tests/test_execute.py b/tests/test_execute.py index e74b5dc24..9b40a5fe3 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -1141,3 +1141,23 @@ def create_file() -> Annotated[Path, path]: assert "3 Collected task" in result.output assert "1 Succeeded" in result.output assert "2 Skipped because unchanged" in result.output + + +def test_download_file(runner, tmp_path): + source = """ + from pathlib import Path + from typing_extensions import Annotated + from upath import UPath + + url = UPath( + "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data" + ) + + def task_download_file(path: UPath = url) -> Annotated[str, Path("data.csv")]: + return path.read_text() + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "1 Succeeded" in result.output From 8552a6c4d5177a607772d64fd87e435afd17176c Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 18 Apr 2024 23:27:29 +0200 Subject: [PATCH 2/3] To changes. --- docs/source/changes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/changes.md b/docs/source/changes.md index 5416f1f0a..1f4c34701 100644 --- a/docs/source/changes.md +++ b/docs/source/changes.md @@ -40,6 +40,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and the value whose state is `None`. - {pull}`594` publishes `NodeLoadError`. - {pull}`596` add project management with rye. +- {pull}`598` replaces requests with httpx. ## 0.4.7 - 2024-03-19 From b24a6d827a34617e196db4c539f3d29c093ec55d Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 18 Apr 2024 23:31:34 +0200 Subject: [PATCH 3/3] Fix. --- pyproject.toml | 1 + tests/test_execute.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 6c81c49c2..7af629bf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ test = [ "pytest-cov", "pytest-xdist", "syrupy", + "aiohttp", ] [project.urls] diff --git a/tests/test_execute.py b/tests/test_execute.py index 9b40a5fe3..d92ce1861 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -1143,6 +1143,7 @@ def create_file() -> Annotated[Path, path]: assert "2 Skipped because unchanged" in result.output +@pytest.mark.end_to_end() def test_download_file(runner, tmp_path): source = """ from pathlib import Path