From d028e044c1d7d398e090e101c8eff8408c8b5100 Mon Sep 17 00:00:00 2001 From: Hendrik Makait Date: Mon, 12 Jun 2023 17:52:35 +0200 Subject: [PATCH 1/6] Rename to get_default_shuffle_method --- distributed/tests/test_client.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py index 43882f6f877..3cc40f1a0c9 100644 --- a/distributed/tests/test_client.py +++ b/distributed/tests/test_client.py @@ -40,12 +40,7 @@ import dask.bag as db from dask import delayed from dask.optimization import SubgraphCallable -from dask.utils import ( - get_default_shuffle_algorithm, - parse_timedelta, - stringify, - tmpfile, -) +from dask.utils import get_default_shuffle_method, parse_timedelta, stringify, tmpfile from distributed import ( CancelledError, @@ -3229,25 +3224,25 @@ def test_default_get(loop_in_thread): # These may change in the future but the selection below should not distributed_default = "p2p" if has_pyarrow else "tasks" local_default = "disk" - assert get_default_shuffle_algorithm() == local_default + assert get_default_shuffle_method() == local_default with Client(s["address"], set_as_default=True, loop=loop) as c: assert dask.base.get_scheduler() == c.get - assert get_default_shuffle_algorithm() == distributed_default + assert get_default_shuffle_method() == distributed_default assert dask.base.get_scheduler() == pre_get - assert get_default_shuffle_algorithm() == local_default + assert get_default_shuffle_method() == local_default c = Client(s["address"], set_as_default=False, loop=loop) assert dask.base.get_scheduler() == pre_get - assert get_default_shuffle_algorithm() == local_default + assert get_default_shuffle_method() == local_default c.close() c = Client(s["address"], set_as_default=True, loop=loop) - assert get_default_shuffle_algorithm() == distributed_default + assert get_default_shuffle_method() == distributed_default assert dask.base.get_scheduler() == c.get c.close() assert dask.base.get_scheduler() == pre_get - assert get_default_shuffle_algorithm() == local_default + assert get_default_shuffle_method() == local_default with Client(s["address"], loop=loop) as c: assert dask.base.get_scheduler() == c.get From 47c4fcdd4e12378acd46904b55389fc6404e4299 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 12 Jun 2023 12:17:32 -0500 Subject: [PATCH 2/6] Temporarily point to dask/dask PR [skip-caching] --- continuous_integration/environment-3.10.yaml | 3 ++- continuous_integration/environment-3.11.yaml | 3 ++- continuous_integration/environment-3.9.yaml | 3 ++- continuous_integration/environment-mindeps.yaml | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/continuous_integration/environment-3.10.yaml b/continuous_integration/environment-3.10.yaml index 86563390f92..a71a4a1b6ad 100644 --- a/continuous_integration/environment-3.10.yaml +++ b/continuous_integration/environment-3.10.yaml @@ -43,7 +43,8 @@ dependencies: - zict # overridden by git tip below - zstandard >=0.9.0 - pip: - - git+https://github.com/dask/dask + # - git+https://github.com/dask/dask + - git+https://github.com/hendrikmakait/dask.git@default-shuffle-ignores-fuse - git+https://github.com/dask/s3fs - git+https://github.com/dask/zict - git+https://github.com/fsspec/filesystem_spec diff --git a/continuous_integration/environment-3.11.yaml b/continuous_integration/environment-3.11.yaml index 5c6eef99828..6d1dcaac39f 100644 --- a/continuous_integration/environment-3.11.yaml +++ b/continuous_integration/environment-3.11.yaml @@ -43,7 +43,8 @@ dependencies: - zict # overridden by git tip below - zstandard >=0.9.0 - pip: - - git+https://github.com/dask/dask + # - git+https://github.com/dask/dask + - git+https://github.com/hendrikmakait/dask.git@default-shuffle-ignores-fuse - git+https://github.com/dask/s3fs - git+https://github.com/dask/zict - git+https://github.com/fsspec/filesystem_spec diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml index ad4d340871e..a12b07a32da 100644 --- a/continuous_integration/environment-3.9.yaml +++ b/continuous_integration/environment-3.9.yaml @@ -50,6 +50,7 @@ dependencies: - zict - zstandard >=0.9.0 - pip: - - git+https://github.com/dask/dask + # - git+https://github.com/dask/dask + - git+https://github.com/hendrikmakait/dask.git@default-shuffle-ignores-fuse - git+https://github.com/dask/crick # Only tested here - keras diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml index 44c5cfdccc4..2db282b38b2 100644 --- a/continuous_integration/environment-mindeps.yaml +++ b/continuous_integration/environment-mindeps.yaml @@ -22,7 +22,8 @@ dependencies: # Distributed depends on the latest version of Dask - pip - pip: - - git+https://github.com/dask/dask + # - git+https://github.com/dask/dask + - git+https://github.com/hendrikmakait/dask.git@default-shuffle-ignores-fuse # test dependencies - pytest - pytest-cov From bd310aacb23336f97871d82ce04ae10cb51f417a Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 12 Jun 2023 15:47:59 -0500 Subject: [PATCH 3/6] gpuCI --- continuous_integration/gpuci/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh index 4568353f2b4..374888db31c 100644 --- a/continuous_integration/gpuci/build.sh +++ b/continuous_integration/gpuci/build.sh @@ -41,7 +41,8 @@ gpuci_logger "Activate conda env" conda activate dask gpuci_logger "Install dask" -python -m pip install git+https://github.com/dask/dask +# python -m pip install git+https://github.com/dask/dask +python -m pip install git+https://github.com/hendrikmakait/dask.git@default-shuffle-ignores-fuse gpuci_logger "Install distributed" python -m pip install -e . From a260ce7ecbba13681d314b9228b4fb319b9a9009 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 12 Jun 2023 15:56:16 -0500 Subject: [PATCH 4/6] pre-commit [skip-caching] --- .pre-commit-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5c030e105f1..dd1e447037c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,5 +63,6 @@ repos: - pytest - tornado - pyarrow - - git+https://github.com/dask/dask + # - git+https://github.com/dask/dask + - git+https://github.com/hendrikmakait/dask.git@default-shuffle-ignores-fuse - git+https://github.com/dask/zict From cec93c4d22dc5845c9c96934dc05146824b4727b Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 12 Jun 2023 16:21:20 -0500 Subject: [PATCH 5/6] gpuCI again [skip-caching] --- continuous_integration/gpuci/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh index 374888db31c..f4e10dc8eb7 100644 --- a/continuous_integration/gpuci/build.sh +++ b/continuous_integration/gpuci/build.sh @@ -40,13 +40,13 @@ gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate dask +gpuci_logger "Install distributed" +python -m pip install -e . + gpuci_logger "Install dask" # python -m pip install git+https://github.com/dask/dask python -m pip install git+https://github.com/hendrikmakait/dask.git@default-shuffle-ignores-fuse -gpuci_logger "Install distributed" -python -m pip install -e . - gpuci_logger "Check Python versions" python --version From 43f746acb591329f15e1e0041179d0001b6cb002 Mon Sep 17 00:00:00 2001 From: Hendrik Makait Date: Tue, 13 Jun 2023 11:41:35 +0200 Subject: [PATCH 6/6] Revert "Temporarily point to dask/dask PR [skip-caching]" This reverts commit 47c4fcdd4e12378acd46904b55389fc6404e4299. --- continuous_integration/environment-3.10.yaml | 3 +-- continuous_integration/environment-3.11.yaml | 3 +-- continuous_integration/environment-3.9.yaml | 3 +-- continuous_integration/environment-mindeps.yaml | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/continuous_integration/environment-3.10.yaml b/continuous_integration/environment-3.10.yaml index a71a4a1b6ad..86563390f92 100644 --- a/continuous_integration/environment-3.10.yaml +++ b/continuous_integration/environment-3.10.yaml @@ -43,8 +43,7 @@ dependencies: - zict # overridden by git tip below - zstandard >=0.9.0 - pip: - # - git+https://github.com/dask/dask - - git+https://github.com/hendrikmakait/dask.git@default-shuffle-ignores-fuse + - git+https://github.com/dask/dask - git+https://github.com/dask/s3fs - git+https://github.com/dask/zict - git+https://github.com/fsspec/filesystem_spec diff --git a/continuous_integration/environment-3.11.yaml b/continuous_integration/environment-3.11.yaml index 6d1dcaac39f..5c6eef99828 100644 --- a/continuous_integration/environment-3.11.yaml +++ b/continuous_integration/environment-3.11.yaml @@ -43,8 +43,7 @@ dependencies: - zict # overridden by git tip below - zstandard >=0.9.0 - pip: - # - git+https://github.com/dask/dask - - git+https://github.com/hendrikmakait/dask.git@default-shuffle-ignores-fuse + - git+https://github.com/dask/dask - git+https://github.com/dask/s3fs - git+https://github.com/dask/zict - git+https://github.com/fsspec/filesystem_spec diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml index a12b07a32da..ad4d340871e 100644 --- a/continuous_integration/environment-3.9.yaml +++ b/continuous_integration/environment-3.9.yaml @@ -50,7 +50,6 @@ dependencies: - zict - zstandard >=0.9.0 - pip: - # - git+https://github.com/dask/dask - - git+https://github.com/hendrikmakait/dask.git@default-shuffle-ignores-fuse + - git+https://github.com/dask/dask - git+https://github.com/dask/crick # Only tested here - keras diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml index 2db282b38b2..44c5cfdccc4 100644 --- a/continuous_integration/environment-mindeps.yaml +++ b/continuous_integration/environment-mindeps.yaml @@ -22,8 +22,7 @@ dependencies: # Distributed depends on the latest version of Dask - pip - pip: - # - git+https://github.com/dask/dask - - git+https://github.com/hendrikmakait/dask.git@default-shuffle-ignores-fuse + - git+https://github.com/dask/dask # test dependencies - pytest - pytest-cov