diff --git a/.github/workflows/tests-macos.yaml b/.github/workflows/tests-macos.yaml deleted file mode 100644 index a89ce4e616e..00000000000 --- a/.github/workflows/tests-macos.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: MacOS tests - -on: - - push - # MacOS tests are currently afflicted by a very high level of flakiness; do not run on - # PR unless this line is uncommented. - # - pull_request - -jobs: - test: - runs-on: macos-latest - - strategy: - fail-fast: false - matrix: - # Many deterministic test failures on Python 3.6 - python-version: ["3.7", "3.8"] - - # Uncomment to stress-test the test suite for random failures - # This will take a LONG time and delay all PRs across the whole github.com/dask! - # run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] - - steps: - - name: Checkout source - uses: actions/checkout@v2 - - - name: Setup Conda Environment - uses: conda-incubator/setup-miniconda@v2 - with: - miniconda-version: latest - python-version: ${{ matrix.python-version }} - environment-file: continuous_integration/environment-${{ matrix.python-version }}.yaml - activate-environment: dask-distributed - auto-activate-base: false - - - name: Cythonize - shell: bash -l {0} - if: ${{ matrix.python-version == '3.7' }} - run: python setup.py build_ext --with-cython - - - name: Install - shell: bash -l {0} - run: python -m pip install --no-deps -e . - - - name: conda list - shell: bash -l {0} - run: conda list - - - name: conda env export - shell: bash -l {0} - run: | - echo -e "--\n--Conda Environment (re-create this with \`conda env create --name -f \`)\n--" - conda env export | grep -E -v '^prefix:.*$' - - - name: Setup SSH - shell: bash -l {0} - run: bash continuous_integration/scripts/setup_ssh.sh - - - name: Test - shell: bash -l {0} - env: - PYTHONFAULTHANDLER: 1 - MARKERS: not avoid_ci - run: | - source continuous_integration/scripts/set_ulimit.sh - pytest distributed -m "not avoid_ci" --runslow - - # - name: Debug with tmate on failure - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d189e291374..c27ae784ea8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -9,11 +9,12 @@ jobs: strategy: fail-fast: false matrix: - # MacOS tests are currently afflicted by a very high level of flakiness. On - # Linux and Windows, run on merge to master and on all pull requests. - # On MacOS, run only on merge to master (see twin workflow tests-macos.yaml). - os: [ubuntu-latest, windows-latest] - python-version: ["3.6", "3.7", "3.8"] + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ["3.6", "3.7", "3.8", "3.9"] + exclude: + # Several failures; Python 3.6 to be phased out soon + - os: macos-latest + python-version: "3.6" # Uncomment to stress-test the test suite for random failures # This will take a LONG time and delay all PRs across the whole github.com/dask! diff --git a/continuous_integration/environment-3.8.yaml b/continuous_integration/environment-3.8.yaml index 153a47d8756..118d4231e4b 100644 --- a/continuous_integration/environment-3.8.yaml +++ b/continuous_integration/environment-3.8.yaml @@ -2,7 +2,6 @@ name: dask-distributed channels: - conda-forge - defaults - - pytorch dependencies: - python=3.8 - pip @@ -11,13 +10,12 @@ dependencies: - click - cloudpickle - dask # overridden by git tip below - - filesystem-spec # overridden by git tip below + - filesystem-spec - h5py - ipykernel - ipywidgets - - joblib # overridden by git tip below + - joblib - jupyter_client - - lz4 # Only tested here - msgpack-python - netcdf4 - paramiko @@ -29,24 +27,16 @@ dependencies: - pytest-repeat - pytest-rerunfailures - pytest-timeout - - python-blosc # Only tested here - - python-snappy # Only tested here - - pytorch # Only tested here - requests - - s3fs # overridden by git tip below + - s3fs - scikit-learn - scipy - sortedcollections - tblib - toolz - - torchvision # Only tested here - tornado=6 - - zict # overridden by git tip below + - zict - zstandard - pip: - git+https://github.com/dask/dask - - git+https://github.com/dask/s3fs - - git+https://github.com/dask/zict - - git+https://github.com/intake/filesystem_spec - - git+https://github.com/joblib/joblib - keras diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml new file mode 100644 index 00000000000..fa0a661a96b --- /dev/null +++ b/continuous_integration/environment-3.9.yaml @@ -0,0 +1,52 @@ +name: dask-distributed +channels: + - conda-forge + - defaults + - pytorch +dependencies: + - python=3.9 + - pip + - asyncssh + - bokeh + - click + - cloudpickle + - dask # overridden by git tip below + - filesystem-spec # overridden by git tip below + - h5py + - ipykernel + - ipywidgets + - joblib # overridden by git tip below + - jupyter_client + - lz4 # Only tested here + - msgpack-python + - netcdf4 + - paramiko + - prometheus_client + - psutil + - pytest + - pytest-asyncio<0.14.0 + - pytest-faulthandler + - pytest-repeat + - pytest-rerunfailures + - pytest-timeout + - python-blosc # Only tested here + - python-snappy # Only tested here + - pytorch # Only tested here + - requests + - s3fs # overridden by git tip below + - scikit-learn + - scipy + - sortedcollections + - tblib + - toolz + - torchvision # Only tested here + - tornado=6 + - zict # overridden by git tip below + - zstandard + - pip: + - git+https://github.com/dask/dask + - git+https://github.com/dask/s3fs + - git+https://github.com/dask/zict + - git+https://github.com/intake/filesystem_spec + - git+https://github.com/joblib/joblib + - keras diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py index f8bbeedfb97..853b9964128 100644 --- a/distributed/cli/tests/test_dask_worker.py +++ b/distributed/cli/tests/test_dask_worker.py @@ -41,7 +41,7 @@ def test_nanny_worker_ports(loop): if d["workers"]: break else: - assert time() - start < 5 + assert time() - start < 60 sleep(0.1) assert ( d["workers"]["tcp://127.0.0.1:9684"]["nanny"] diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py index ed7cc299dec..9c50ae50434 100644 --- a/distributed/dashboard/tests/test_scheduler_bokeh.py +++ b/distributed/dashboard/tests/test_scheduler_bokeh.py @@ -15,9 +15,10 @@ import dask from dask.core import flatten from dask.utils import stringify -from distributed.utils import format_dashboard_link from distributed.client import wait +from distributed.compatibility import MACOS from distributed.metrics import time +from distributed.utils import format_dashboard_link from distributed.utils_test import gen_cluster, inc, dec, slowinc, div, get_cert from distributed.dashboard.components.worker import Counters from distributed.dashboard.scheduler import applications @@ -749,6 +750,7 @@ async def test_aggregate_action(c, s, a, b): assert ("compute") in mbk.action_source.data["names"] +@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS) @gen_cluster(client=True, scheduler_kwargs={"dashboard": True}) async def test_compute_per_key(c, s, a, b): mbk = ComputePerKey(s) diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py index 5f84b3165b4..e747cf95a20 100644 --- a/distributed/deploy/tests/test_adaptive.py +++ b/distributed/deploy/tests/test_adaptive.py @@ -302,7 +302,7 @@ async def test_adapt_down(): start = time() while len(cluster.scheduler.workers) != 2: await asyncio.sleep(0.1) - assert time() < start + 3 + assert time() < start + 60 @gen_test(timeout=30) diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py index 7a0f60ba011..afdc10cbe04 100644 --- a/distributed/deploy/tests/test_spec_cluster.py +++ b/distributed/deploy/tests/test_spec_cluster.py @@ -196,6 +196,7 @@ async def test_unexpected_closed_worker(cleanup): assert len(cluster.workers) == 2 +@pytest.mark.flaky(reruns=10, reruns_delay=5) @pytest.mark.slow @pytest.mark.asyncio async def test_restart(cleanup): diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py index 376194549ce..0bea6f7dc75 100644 --- a/distributed/deploy/tests/test_ssh.py +++ b/distributed/deploy/tests/test_ssh.py @@ -81,7 +81,7 @@ def test_defer_to_old(loop): @pytest.mark.avoid_ci -def test_old_ssh_wih_local_dir(loop): +def test_old_ssh_with_local_dir(loop): with pytest.warns(Warning): from distributed.deploy.old_ssh import SSHCluster as OldSSHCluster diff --git a/distributed/utils.py b/distributed/utils.py index d7d4db6b2c5..502118218c7 100644 --- a/distributed/utils.py +++ b/distributed/utils.py @@ -64,6 +64,12 @@ except ImportError: thread_state = threading.local() +# For some reason this is required in python >= 3.9 +if WINDOWS: + import multiprocessing.popen_spawn_win32 +else: + import multiprocessing.popen_spawn_posix + logger = _logger = logging.getLogger(__name__) diff --git a/setup.py b/setup.py index 9160e1ffdc9..5483e0b963d 100755 --- a/setup.py +++ b/setup.py @@ -92,6 +92,7 @@ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Topic :: Scientific/Engineering", "Topic :: System :: Distributed Computing", ],