diff --git a/.github/workflows/additional.yml b/.github/workflows/additional.yml new file mode 100644 index 00000000000..92e21c77ae3 --- /dev/null +++ b/.github/workflows/additional.yml @@ -0,0 +1,86 @@ +name: Additional + +on: [push, pull_request] + +jobs: + mindeps: + runs-on: "ubuntu-latest" + timeout-minutes: 180 + + steps: + - name: Checkout source + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup Conda Environment + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + condarc-file: continuous_integration/condarc + use-mamba: true + python-version: 3.7 + environment-file: continuous_integration/environment-mindeps.yaml + activate-environment: dask-distributed + + - name: Show conda options + shell: bash -l {0} + run: conda config --show + + - name: Install + shell: bash -l {0} + run: python -m pip install --no-deps -e . + + - name: mamba list + shell: bash -l {0} + run: mamba list + + - name: mamba env export + shell: bash -l {0} + run: | + echo -e "--\n--Conda Environment (re-create this with \`mamba env create --name -f \`)\n--" + mamba env export | grep -E -v '^prefix:.*$' + + - name: Setup SSH + shell: bash -l {0} + run: bash continuous_integration/scripts/setup_ssh.sh + + - name: Reconfigure pytest-timeout + shell: bash -l {0} + run: sed -i.bak 's/timeout_method = thread/timeout_method = signal/' setup.cfg + + - name: Test + shell: bash -l {0} + env: + PYTHONFAULTHANDLER: 1 + # FIXME ipv6-related failures on Ubuntu github actions CI + # https://github.com/dask/distributed/issues/4514 + DISABLE_IPV6: 1 + run: | + source continuous_integration/scripts/set_ulimit.sh + pytest distributed -m "not avoid_ci" --runslow \ + --junitxml reports/pytest.xml -o junit_suite_name=mindeps --cov=distributed --cov-report=xml + + - name: Coverage + uses: codecov/codecov-action@v1 + + - name: Upload test artifacts + # ensure this runs even if pytest fails + if: > + always() && + (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure') + uses: actions/upload-artifact@v2 + with: + name: ${{ env.mindeps }} + path: reports + + - name: Upload timeout reports + # ensure this runs even if pytest fails + if: > + always() && + (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure') + uses: actions/upload-artifact@v2 + with: + name: ${{ env.mindeps }}-timeouts + path: test_timeout_dump diff --git a/continuous_integration/environment-mindeps.yaml b/continuous_integration/environment-mindeps.yaml new file mode 100644 index 00000000000..9c22933388b --- /dev/null +++ b/continuous_integration/environment-mindeps.yaml @@ -0,0 +1,31 @@ +name: dask-distributed +channels: + - conda-forge + - defaults +dependencies: + - python=3.7 + - click=6.7 + - cloudpickle=1.5.0 + - jinja2 + - msgpack-python=0.6.0 + - packaging=20.0 + - psutil=5.4.7 + - sortedcontainers=2.0.4 + - tblib=1.6.0 + - toolz=0.10.0 + - tornado=5 + - zict=0.1.3 + - pyyaml + - setuptools + # test dependencies + - pytest + - pytest-asyncio<0.14.0 + - pytest-cov + - pytest-faulthandler + - pytest-repeat + - pytest-rerunfailures + - pytest-timeout + - pip + - pip: + # Distributed depends on the latest version of Dask + - git+https://github.com/dask/dask \ No newline at end of file diff --git a/distributed/compatibility.py b/distributed/compatibility.py index ad8964120a3..9d6d26cbe57 100644 --- a/distributed/compatibility.py +++ b/distributed/compatibility.py @@ -3,6 +3,7 @@ import logging import platform import sys +from distutils.version import LooseVersion import tornado @@ -10,6 +11,7 @@ logging_names.update(logging._levelToName) # type: ignore logging_names.update(logging._nameToLevel) # type: ignore +PY_VERSION = LooseVersion(".".join(map(str, sys.version_info[:3]))) PYPY = platform.python_implementation().lower() == "pypy" LINUX = sys.platform == "linux" MACOS = sys.platform == "darwin" diff --git a/distributed/protocol/tests/test_collection_cuda.py b/distributed/protocol/tests/test_collection_cuda.py index 22be1c66a08..403391f3b10 100644 --- a/distributed/protocol/tests/test_collection_cuda.py +++ b/distributed/protocol/tests/test_collection_cuda.py @@ -1,9 +1,6 @@ import pytest pytestmark = pytest.mark.gpu - -from dask.dataframe.utils import assert_eq - from distributed.protocol import deserialize, serialize @@ -44,6 +41,7 @@ def test_serialize_cupy(collection, y, y_serializer): def test_serialize_pandas_pandas(collection, df2, df2_serializer): cudf = pytest.importorskip("cudf") pd = pytest.importorskip("pandas") + dd = pytest.importorskip("dask.dataframe") df1 = cudf.DataFrame({"A": [1, 2, None], "B": [1.0, 2.0, None]}) if df2 is not None: df2 = cudf.from_pandas(pd.DataFrame(df2)) @@ -61,8 +59,8 @@ def test_serialize_pandas_pandas(collection, df2, df2_serializer): assert sub_headers[1]["serializer"] == df2_serializer assert isinstance(t, collection) - assert_eq(t["df1"] if isinstance(t, dict) else t[0], df1) + dd.assert_eq(t["df1"] if isinstance(t, dict) else t[0], df1) if df2 is None: assert (t["df2"] if isinstance(t, dict) else t[1]) is None else: - assert_eq(t["df2"] if isinstance(t, dict) else t[1], df2) + dd.assert_eq(t["df2"] if isinstance(t, dict) else t[1], df2) diff --git a/distributed/protocol/tests/test_highlevelgraph.py b/distributed/protocol/tests/test_highlevelgraph.py index 518dceb66f6..77bd8848112 100644 --- a/distributed/protocol/tests/test_highlevelgraph.py +++ b/distributed/protocol/tests/test_highlevelgraph.py @@ -2,6 +2,11 @@ import pytest +np = pytest.importorskip("numpy") +pd = pytest.importorskip("pandas") + +from numpy.testing import assert_array_equal + import dask import dask.array as da import dask.dataframe as dd @@ -9,11 +14,6 @@ from distributed.diagnostics import SchedulerPlugin from distributed.utils_test import gen_cluster -np = pytest.importorskip("numpy") -pd = pytest.importorskip("pandas") - -from numpy.testing import assert_array_equal - @gen_cluster(client=True) async def test_combo_of_layer_types(c, s, a, b): diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py index 436eb78ad5d..1f1de5d2fc6 100644 --- a/distributed/protocol/tests/test_pickle.py +++ b/distributed/protocol/tests/test_pickle.py @@ -1,21 +1,21 @@ import gc -import sys import weakref from functools import partial from operator import add import pytest +from distributed.compatibility import PY_VERSION from distributed.protocol import deserialize, serialize from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads -if sys.version_info < (3, 8): +if PY_VERSION < "3.8": try: import pickle5 as pickle except ImportError: - import pickle + import pickle # type: ignore else: - import pickle + import pickle # type: ignore class MemoryviewHolder: diff --git a/distributed/protocol/tests/test_protocol_utils.py b/distributed/protocol/tests/test_protocol_utils.py index 5ebcb6e1e12..912f041e097 100644 --- a/distributed/protocol/tests/test_protocol_utils.py +++ b/distributed/protocol/tests/test_protocol_utils.py @@ -66,6 +66,7 @@ def test_readonly_buffer(self): assert result == base def test_catch_non_memoryview(self): + pytest.importorskip("numpy") with pytest.raises(TypeError, match="Expected memoryview"): merge_memoryviews([b"1234", memoryview(b"4567")]) diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py index a36b787ac6b..15dd84a6050 100644 --- a/distributed/tests/test_client.py +++ b/distributed/tests/test_client.py @@ -7210,7 +7210,7 @@ def _verify_cluster_dump(path, _format: str, addresses: set[str]) -> dict: path += ".msgpack.gz" with gzip.open(path) as fd_zip: - state = msgpack.unpack(fd_zip) + state = msgpack.unpack(fd_zip, raw=False) else: import yaml diff --git a/requirements.txt b/requirements.txt index 35c11a74843..032a0b19b2d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ -click >= 6.6 +click >= 6.7 cloudpickle >= 1.5.0 dask == 2021.12.0 jinja2 msgpack >= 0.6.0 packaging >= 20.0 -psutil >= 5.0 -sortedcontainers !=2.0.0, !=2.0.1 +psutil >= 5.4.7 +sortedcontainers >= 2.0.4 tblib >= 1.6.0 -toolz >= 0.8.2 +toolz >= 0.10.0 tornado >= 5;python_version<'3.8' tornado >= 6.0.3;python_version>='3.8' zict >= 0.1.3