Add minimum dependencies CI build

.github/workflows/additional.yml

@@ -0,0 +1,86 @@
+name: Additional
+
+on: [push, pull_request]
+
+jobs:
+  mindeps:
+    runs-on: "ubuntu-latest"
+    timeout-minutes: 180
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          condarc-file: continuous_integration/condarc
+          use-mamba: true
+          python-version: 3.7
+          environment-file: continuous_integration/environment-mindeps.yaml
+          activate-environment: dask-distributed
+
+      - name: Show conda options
+        shell: bash -l {0}
+        run: conda config --show
+
+      - name: Install
+        shell: bash -l {0}
+        run: python -m pip install --no-deps -e .
+
+      - name: mamba list
+        shell: bash -l {0}
+        run: mamba list
+
+      - name: mamba env export
+        shell: bash -l {0}
+        run: |
+          echo -e "--\n--Conda Environment (re-create this with \`mamba env create --name <name> -f <output_file>\`)\n--"
+          mamba env export | grep -E -v '^prefix:.*$'
+
+      - name: Setup SSH
+        shell: bash -l {0}
+        run: bash continuous_integration/scripts/setup_ssh.sh
+
+      - name: Reconfigure pytest-timeout
+        shell: bash -l {0}
+        run: sed -i.bak 's/timeout_method = thread/timeout_method = signal/' setup.cfg
+
+      - name: Test
+        shell: bash -l {0}
+        env:
+          PYTHONFAULTHANDLER: 1
+          # FIXME ipv6-related failures on Ubuntu github actions CI
+          # https://github.com/dask/distributed/issues/4514
+          DISABLE_IPV6: 1
+        run: |
+          source continuous_integration/scripts/set_ulimit.sh
+          pytest distributed -m "not avoid_ci" --runslow  \
+            --junitxml reports/pytest.xml -o junit_suite_name=mindeps --cov=distributed --cov-report=xml
+
+      - name: Coverage
+        uses: codecov/codecov-action@v1
+
+      - name: Upload test artifacts
+        # ensure this runs even if pytest fails
+        if: >
+          always() &&
+          (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure')
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.mindeps }}
+          path: reports
+
+      - name: Upload timeout reports
+        # ensure this runs even if pytest fails
+        if: >
+          always() &&
+          (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure')
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.mindeps }}-timeouts
+          path: test_timeout_dump

continuous_integration/environment-mindeps.yaml

@@ -0,0 +1,31 @@
+name: dask-distributed
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.7
+  - click=6.7
+  - cloudpickle=1.5.0
+  - jinja2
+  - msgpack-python=0.6.0
+  - packaging=20.0
+  - psutil=5.4.7
+  - sortedcontainers=2.0.4
+  - tblib=1.6.0
+  - toolz=0.10.0
+  - tornado=5
+  - zict=0.1.3
+  - pyyaml
+  - setuptools
+  # test dependencies
+  - pytest
+  - pytest-asyncio<0.14.0
+  - pytest-cov
+  - pytest-faulthandler
+  - pytest-repeat
+  - pytest-rerunfailures
+  - pytest-timeout
+  - pip
+  - pip:
+    # Distributed depends on the latest version of Dask
+    - git+https://github.com/dask/dask

distributed/compatibility.py

@@ -3,13 +3,15 @@
 import logging
 import platform
 import sys
+from distutils.version import LooseVersion
 
 import tornado
 
 logging_names: dict[str | int, int | str] = {}
 logging_names.update(logging._levelToName)  # type: ignore
 logging_names.update(logging._nameToLevel)  # type: ignore
 
+PY_VERSION = LooseVersion(".".join(map(str, sys.version_info[:3])))
 PYPY = platform.python_implementation().lower() == "pypy"
 LINUX = sys.platform == "linux"
 MACOS = sys.platform == "darwin"

distributed/protocol/tests/test_collection_cuda.py

@@ -1,9 +1,6 @@
 import pytest
 
 pytestmark = pytest.mark.gpu
-
-from dask.dataframe.utils import assert_eq
-
 from distributed.protocol import deserialize, serialize
 
 
@@ -44,6 +41,7 @@ def test_serialize_cupy(collection, y, y_serializer):
 def test_serialize_pandas_pandas(collection, df2, df2_serializer):
     cudf = pytest.importorskip("cudf")
     pd = pytest.importorskip("pandas")
+    dd = pytest.importorskip("dask.dataframe")
     df1 = cudf.DataFrame({"A": [1, 2, None], "B": [1.0, 2.0, None]})
     if df2 is not None:
         df2 = cudf.from_pandas(pd.DataFrame(df2))
@@ -61,8 +59,8 @@ def test_serialize_pandas_pandas(collection, df2, df2_serializer):
     assert sub_headers[1]["serializer"] == df2_serializer
     assert isinstance(t, collection)
 
-    assert_eq(t["df1"] if isinstance(t, dict) else t[0], df1)
+    dd.assert_eq(t["df1"] if isinstance(t, dict) else t[0], df1)
     if df2 is None:
         assert (t["df2"] if isinstance(t, dict) else t[1]) is None
     else:
-        assert_eq(t["df2"] if isinstance(t, dict) else t[1], df2)
+        dd.assert_eq(t["df2"] if isinstance(t, dict) else t[1], df2)

distributed/protocol/tests/test_highlevelgraph.py

@@ -2,18 +2,18 @@
 
 import pytest
 
+np = pytest.importorskip("numpy")
+pd = pytest.importorskip("pandas")
+
+from numpy.testing import assert_array_equal
+
 import dask
 import dask.array as da
 import dask.dataframe as dd
 
 from distributed.diagnostics import SchedulerPlugin
 from distributed.utils_test import gen_cluster
 
-np = pytest.importorskip("numpy")
-pd = pytest.importorskip("pandas")
-
-from numpy.testing import assert_array_equal
-
 
 @gen_cluster(client=True)
 async def test_combo_of_layer_types(c, s, a, b):

distributed/protocol/tests/test_pickle.py

@@ -1,21 +1,21 @@
 import gc
-import sys
 import weakref
 from functools import partial
 from operator import add
 
 import pytest
 
+from distributed.compatibility import PY_VERSION
 from distributed.protocol import deserialize, serialize
 from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads
 
-if sys.version_info < (3, 8):
+if PY_VERSION < "3.8":
     try:
         import pickle5 as pickle
     except ImportError:
-        import pickle
+        import pickle  # type: ignore
 else:
-    import pickle
+    import pickle  # type: ignore
 
 
 class MemoryviewHolder:

distributed/protocol/tests/test_protocol_utils.py

@@ -66,6 +66,7 @@ def test_readonly_buffer(self):
         assert result == base
 
     def test_catch_non_memoryview(self):
+        pytest.importorskip("numpy")
         with pytest.raises(TypeError, match="Expected memoryview"):
             merge_memoryviews([b"1234", memoryview(b"4567")])
 

distributed/tests/test_client.py

@@ -7210,7 +7210,7 @@ def _verify_cluster_dump(path, _format: str, addresses: set[str]) -> dict:
         path += ".msgpack.gz"
 
         with gzip.open(path) as fd_zip:
-            state = msgpack.unpack(fd_zip)
+            state = msgpack.unpack(fd_zip, raw=False)
     else:
         import yaml
 

requirements.txt

@@ -1,13 +1,13 @@
-click >= 6.6
+click >= 6.7
 cloudpickle >= 1.5.0
 dask == 2021.12.0
 jinja2
 msgpack >= 0.6.0
 packaging >= 20.0
-psutil >= 5.0
-sortedcontainers !=2.0.0, !=2.0.1
+psutil >= 5.4.7
+sortedcontainers >= 2.0.4
 tblib >= 1.6.0
-toolz >= 0.8.2
+toolz >= 0.10.0
 tornado >= 5;python_version<'3.8'
 tornado >= 6.0.3;python_version>='3.8'
 zict >= 0.1.3